python 日志入库mongodb

今天和大家分享一个python入库mongodb的脚本。。。

涉及到python和mongodb，那么安装相应的模块四必不可少的，最简单的安装方法，或者非pip不可了。

# pip install pymongo==3.0.4

顺便也记录下源码安装的方式

# wget https://pypi.python.org/packages/source/p/pymongo/pymongo-2.8.tar.gz#md5=23100361c9af1904eb2d7722f2658114 --no-check-certificate
# tar xf pymongo-2.8.tar.gz
# cd pymongo-2.8
# python setup.py install

摘自一则日志

35783	s100	android	47	5	192.168.1.100	2015-09-05 08:03:19	strengthenHeroByHeroes	{"consume_gold":{"ogold":2893821,"cgold":1700,"gold":2892121,"tag":"strengthenHeroByHeroes"},"taskInfo":[{"id":2310033,"progress":2,"status":0}],"delHeroList":{"id":102014,"id":102014,"id":102014,"id":102010,"id":102010},"id":100026,"olevel":46,"oexp":1700,"cexp":1700,"level":46,"exp":3400}	865982021462182		XiaoMi

入库mongodb的python脚本

#!/usr/bin/env python
#coding:utf8

import os
import sys
import json
from pymongo import MongoClient
from datetime import datetime, date, time

def ConMongo(host,port,cur_db,username,password):	#链接MongoDB
    client = MongoClient(host,port)
    db = client[cur_db]
    db.authenticate(username,password)
    table = db.gamelogs
    return table

def parseLog(file_log,Connection):
    dic = {}
    dl = []
    with open(file_log) as fd:
        for line in fd:
	    try:
	        tokens = line.strip().split(‘\t‘)
	        uid = tokens[0]
	        server = tokens[1]
	        system = tokens[2]
	        level = int(tokens[3])
	        vip_level = tokens[4]
	        ip = tokens[5]
	        time = datetime.strptime(tokens[6], "%Y-%m-%d %H:%M:%S")		#将时间字符串转换成时间格式
	        action = tokens[7]
	        result = json.loads(tokens[8])						#特殊字符串转换成json格式
	        uuid = tokens[9]
		if uid == ‘undefined‘:
		    if result["game_user_id"]:
			uid = result["game_user_id"]
		if len(tokens) == 12:
	            channel = tokens[11]
		else:
		    channel = ‘‘
	        dic = {‘uid‘:uid,‘server‘:server,‘system‘:system,‘level‘:level,‘vip_level‘:vip_level,‘ip‘:ip,‘time‘:time,‘action‘:action,‘result‘:result,‘uuid‘:uuid,‘channel‘:channel}
		dl.append(dic)
		if len(dl) == 20000:
		    Connection.insert_many(dl)
		    dl = []
	    except Exception,e:
		print e, line
	if len(dl) > 0:
	    Connection.insert_many(dl)

if __name__ == ‘__main__‘:
    Conn = ConMongo(‘localhost‘,27017,‘talefundb‘,‘talefun‘,‘123456‘)
    try:
        parseLog(sys.argv[1],Conn)
    except IndexError,e:
	print ‘./%s path_logfile‘ % os.path.basename(__file__)