1 # -*- coding=utf-8 -*-
2 #curl公共程序
3 #张明伟 20200331
4 import pycurl
5 from io import BytesIO
6 import datetime,time,os
7 import logging
8 import json
9 from logging import handlers
10 testdebug=1 #测试模式：1为是，0为否
11 if testdebug:
12 ip = ‘221.122.125.207‘ #测试环境IP
13 else:
14 ip = ‘221.122.125.201‘ #正式环境IP
15
16 pth=os.getcwd()+os.sep+‘data‘+os.sep
17 logpath =os.getcwd() + os.sep + ‘logs‘ + os.sep
18
19 #返回当前日期
20 def dtnow():
21 return datetime.datetime.today().strftime(‘%Y%m%d‘)
22
23 #返回文件名相关信息
24 def filestr(filename):
25 filepath,tmpfile=os.path.split(filename)
26 basename,extention=os.path.splitext(tmpfile)
27 return filepath,tmpfile,basename,extention
28
29 #暂时不用
30 def init():
31 dt=(datetime.datetime.today()+datetime.timedelta(days=-1)).strftime(‘%Y-%m-%d‘)
32 rdt=dt.replace(‘-‘,‘‘)
33 logfile=‘bigdata‘+rdt+‘.log‘
34 logging.basicConfig(level=logging.INFO,format=‘%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s‘,filename=logfile)
35 logging.info("系统金额我发你")
36 print(os.path.exists(logfile))
37
38 #curlcmd用于上传下载，scp文件，http请求
39 #op=download/upload 下载或上传 默认是下载，上传时需要传入filename,http请求时需要data
40 def curlcmd(url,op="download",filename=None,data=""):
41 if not filename:
42 filename=‘‘
43 c = pycurl.Curl()
44 c.setopt(pycurl.SSL_VERIFYPEER, False) #ssl不信任或无证书时需要设置这个 等同于curl -k/--insecure 参数
45 c.setopt(pycurl.SSL_VERIFYHOST, False) #
46 c.setopt(pycurl.URL, url+filename) #上传时需要指定上传文件名，所以filename加入到url中
47 c.fp = BytesIO() #设置输出
48 c.setopt(pycurl.POST, 1) #以POST方式提交
49 # c.setopt(pycurl.POSTFIELDS, urllib.urlencode(data))
50 if isinstance(data,list): #提交大量数据
51 c.setopt(pycurl.HTTPPOST, data)
52 elif isinstance(data,dict): #json格式的数据上传 对应着大量的表单数据提交
53 c.setopt(pycurl.POSTFIELDS,json.dumps(data))
54 c.setopt(pycurl.USERPWD, "ftpuser:[email protected]") #用户名密码
55 c.setopt(pycurl.WRITEFUNCTION, c.fp.write) #curl执行结果
56
57 #print(data,url,op,filename)
58 if op=="upload": #上传文件
59 if not filename:
60 raise Exception("请输入上传文件名!")
61 filename1=pth+filename
62 filename1size=os.path.getsize(filename1)
63 c.setopt(pycurl.UPLOAD, True) #设置上传标志
64 c.setopt(pycurl.INFILESIZE, filename1size) #上传需要传入文件大小标志
65 # c.setopt(pycurl.READFUNCTION, open(filename1,"rd").read(filename1size))
66 c.setopt(pycurl.READFUNCTION, open(filename1, "rb").read)
67 # print(data, ‘@@@@@@‘)
68 # print(urllib.urlencode(data))
69 c.perform() #curl中真正的执行
70 code = c.getinfo(c.HTTP_CODE) #获取返回码
71 html = c.fp.getvalue() #获取返回结果
72 print(html)
73 return {‘code‘:code,‘html‘:html} #返回json格式
74
75 #日志类
76 class Logger(object):
77 level_relations={
78 ‘debug‘:logging.DEBUG,
79 ‘info‘:logging.INFO,
80 ‘waining‘:logging.WARNING,
81 ‘error‘:logging.ERROR,
82 ‘crit‘:logging.CRITICAL
83 }
84
85 def __init__(self,filename=None,level=‘debug‘,when=‘D‘,backCount=3,fmt=‘%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s‘):
86 if not os.path.exists(logpath):
87 os.mkdir(logpath)
88 if filename is None:
89 filename=filestr(__file__)[2] #返回无后缀的文件名
90 logfile=logpath+filename+dtnow()+‘.log‘
91 self.logger=logging.getLogger(logfile)
92 format_str=logging.Formatter(fmt)
93 self.logger.setLevel(self.level_relations.get(level))
94 sh=logging.StreamHandler()
95 sh.setFormatter(format_str)
96 th=handlers.TimedRotatingFileHandler(filename=logfile,when=when,backupCount=backCount)
97 th.setFormatter(format_str)
98 self.logger.addHandler(sh)
99 self.logger.addHandler(th)
100
101 #json数据类型解析
102 class expressDict():
103 fldlist = [] # type:list #字段名列表
104 valuelist = [] # type:list #值列表
105 sql = "" #SQL INSERT语句列表
106 valuesql = "" #SQL值语句列表
107
108 def __init__(self,jsstr,sqlquote="‘"):
109 self.jsstr = jsstr
110 self.sqlquote = "‘"
111 self.tabname = ""
112 self.sqls=[]
113 self.sqlFlag = True #默认是需要得到SQL语句
114 # self.fldlist=[] #type:list
115 # self.valuelist=[] #type:list
116 # self.sql = ""
117 # self.valuesql = ""
118
119 #数据处理后再次重置数据
120 def __clear(self):
121 self.fldlist = [] # type:list
122 self.valuelist = [] # type:list
123 self.sql = ""
124 self.valuesql = ""
125
126 def __display_item(self, jsstr):
127 """
128 根据json格式自动生成SQL语句或文件:递归
129 文件存放在{pth}目录下
130 sqlFlag默认为真，为真时产生sql语句，否则生成脚本
131 :param jsstr:
132 :return:
133 """
134 #文件格式存放时不需要分割符
135 if not self.sqlFlag:
136 self.sqlquote=""
137 #如果json格式中数据库字典格式，则把键值做为表名
138 if isinstance(jsstr, dict):
139 for k,v in jsstr.items():
140 if not isinstance(v,list):
141 self.fldlist.append(k)
142 self.valuelist.append(v)
143 # print(k,‘--->‘,v)
144 self.sql=self.sql+k+‘,‘
145 self.valuesql=self.valuesql+self.sqlquote+v+self.sqlquote+‘,‘
146 else:
147 if not k:
148 # print(self.tabname,"*" * 20)
149 pass
150 else:
151 if k == self.tabname:
152 self.sql = self.sql+‘) ‘
153 self.tabname=k
154 # print(self.tabname+‘@@@@@@@‘)
155 self.__display_item(v)
156 # 如果json格式中数据库列表格式，则把其他数据做库数据值
157 elif isinstance(jsstr, list):
158 for jslst in jsstr:
159 if self.sqlFlag:
160 self.sql=self.sql+"insert into %s (" %(self.tabname)
161 self.__display_item(jslst)
162 self.sql = self.sql.strip(‘,‘) + ") values ("+self.valuesql.strip(‘,‘)+");"
163 self.sqls.append(self.sql)
164 self.__clear()
165 else:
166 finame = pth + self.tabname + ‘_‘ + dtnow() + ‘.del‘
167 self.__display_item(jslst)
168 data = self.valuesql.strip(‘,‘) + os.linesep # type:str
169 self.__clear()
170 print(finame, data)
171 with open(finame, ‘a+‘) as fp:
172 fp.write(data.encode("utf-8"))
173
174 def display_items(self,sqlFlag=True):
175 # return self.display_item(self.jsstr)
176 self.sqlFlag=sqlFlag
177 return self.__display_item(self.jsstr)
178
179 def dispay_sqls(self):
180 for sql in self.sqls:
181 print(sql)
182 # print(self.valuesql)
183
184 # curl -k -v -T "{table_name1_20200403.del,table_name2_20200403.del}" -u ftpuser scp://21.12.2.196/tmp/
185 # print(scptools(url="scp://21.12.2.196//tmp/gjj20200402.log"))
186 # print(scptools(url="scp://[email protected]//tmp/"))
187 # print(curlcmd(data="",url="scp://21.12.2.196//tmp/gjj20200402.log"))
188 # print(curlcmd(data="",url="scp://[email protected]/tmp/",op="upload",filename="test111.txt"))
原文地址：https://www.cnblogs.com/silencemaker/p/12630679.html
时间： 2024-10-07 19:31:52