Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/writequery.py : 7%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*Execute a MySQL write query on a database table*
6:Author:
7 David Young
9:Date Created:
10 June 21, 2016
11"""
12################# GLOBAL IMPORTS ####################
13from builtins import str
14import sys
15import os
16os.environ['TERM'] = 'vt100'
17from fundamentals import tools
18import time
21def writequery(
22 log,
23 sqlQuery,
24 dbConn,
25 Force=False,
26 manyValueList=False
27):
28 """*Execute a MySQL write command given a sql query*
30 **Key Arguments:**
31 - ``sqlQuery`` -- the MySQL command to execute
32 - ``dbConn`` -- the db connection
33 - ``Force`` -- do not exit code if error occurs, move onto the next command
34 - ``manyValueList`` -- a list of value tuples if executing more than one insert
36 **Return:**
37 - ``message`` -- error/warning message
39 **Usage:**
41 Here's an example of how to create a table using the database connection passed to the function:
43 .. code-block:: python
45 from fundamentals.mysql import writequery
46 sqlQuery = "CREATE TABLE `testing_table` (`id` INT NOT NULL, PRIMARY KEY (`id`))"
47 message = writequery(
48 log=log,
49 sqlQuery=sqlQuery,
50 dbConn=dbConn,
51 Force=False,
52 manyValueList=False
53 )
55 Here's a many value insert example:
57 .. code-block:: python
59 from fundamentals.mysql import writequery
60 sqlQuery = "INSERT INTO testing_table (id) values (%s)"
61 message = writequery(
62 log=log,
63 sqlQuery=sqlQuery,
64 dbConn=dbConn,
65 Force=False,
66 manyValueList=[(1,), (2,), (3,), (4,), (5,), (6,), (7,),
67 (8,), (9,), (10,), (11,), (12,), ]
68 )
70 """
71 log.debug('starting the ``writequery`` function')
72 import pymysql
73 import warnings
74 warnings.filterwarnings('error', category=pymysql.Warning)
75 message = ""
76 try:
77 cursor = dbConn.cursor(pymysql.cursors.DictCursor)
78 except Exception as e:
79 log.error('could not create the database cursor.')
80 # EXECUTE THE SQL COMMAND
82 try:
83 if manyValueList == False:
84 cursor.execute(sqlQuery)
86 else:
87 # cursor.executemany(sqlQuery, manyValueList)
88 # INSET LARGE LISTS IN BATCHES TO STOP MYSQL SERVER BARFING
89 batch = 100000
90 offset = 0
91 stop = 0
93 while stop == 0:
94 thisList = manyValueList[offset:offset + batch]
95 offset += batch
96 a = len(thisList)
97 cursor.executemany(sqlQuery, thisList)
98 dbConn.commit()
99 if len(thisList) < batch:
100 stop = 1
101 except pymysql.err.ProgrammingError as e:
102 message = 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % (sqlQuery,
103 str(e))
104 if Force == False:
105 log.error(message)
106 raise
107 else:
108 log.warning(message)
109 except pymysql.Error as e:
111 try:
112 e = e.args
113 except:
114 pass
116 if e[0] == 1050 and 'already exists' in e[1]:
117 log.info(str(e) + '\n')
118 elif e[0] == 1062:
119 # Duplicate Key error
120 log.debug('Duplicate Key error: %s\n' % (str(e), ))
121 message = "duplicate key error"
122 elif e[0] == 1061:
123 # Duplicate Key error
124 log.debug('index already exists: %s\n' % (str(e), ))
125 message = "index already exists"
126 elif "Duplicate entry" in str(e):
127 log.debug('Duplicate Key error: %s\n' % (str(e), ))
128 message = "duplicate key error"
129 elif "Deadlock" in str(e):
130 i = 0
131 while i < 10:
132 time.sleep(1)
133 i += 1
134 try:
135 if manyValueList == False:
136 cursor.execute(sqlQuery)
137 else:
138 # cursor.executemany(sqlQuery, manyValueList)
139 # INSET LARGE LISTS IN BATCHES TO STOP MYSQL SERVER
140 # BARFING
141 batch = 100000
142 offset = 0
143 stop = 0
145 while stop == 0:
146 thisList = manyValueList[offset:offset + batch]
147 offset += batch
148 a = len(thisList)
149 cursor.executemany(sqlQuery, thisList)
150 dbConn.commit()
151 if len(thisList) < batch:
152 stop = 1
153 i = 20
154 except:
155 pass
156 if i == 10:
157 log.error('Deadlock: %s\n' % (str(e), ))
158 message = "Deadlock error"
159 raise
161 else:
162 message = 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % (sqlQuery,
163 str(e))
164 if Force == False:
165 log.error(message)
166 raise
167 else:
168 log.warning(message)
170 except pymysql.Warning as e:
171 log.info(str(e))
172 except Exception as e:
173 if "truncated" in str(e):
174 log.error('%s\n Here is the sqlquery:\n%s\n' % (str(e), sqlQuery))
175 if manyValueList:
176 log.error('... and the values:\n%s\n' % (thisList, ))
177 elif "Duplicate entry" in str(e):
178 log.warning('Duplicate Key error: %s\n' % (str(e), ))
179 message = "duplicate key error"
180 else:
181 log.error(
182 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' %
183 (sqlQuery, str(e)))
184 if Force == False:
185 sys.exit(0)
186 cursor.close()
187 return -1
188 dbConn.commit()
189 # CLOSE THE CURSOR
190 cOpen = True
191 count = 0
192 while cOpen:
193 try:
194 cursor.close()
195 cOpen = False
196 except Exception as e:
197 time.sleep(1)
198 count += 1
199 if count == 10:
200 log.warning('could not close the db cursor ' + str(e) + '\n')
201 raise e
202 count = 0
204 log.debug('completed the ``writequery`` function')
205 return message