python数据统计脚本实例mysql,redis

#!/usr/bin/env python#-*-coding:utf-8-*-import redisimport sysimport MySQLdbimport timeimport osimport os.pathimport refrom decimal import *######################MySQLdb开始#####

chengxuyuanyonghu

1167人浏览 · 2017-08-24 16:02:49

chengxuyuanyonghu · 2017-08-24 16:02:49 发布

#!/usr/bin/env python
#-*-coding:utf-8-*-

import redis
import sys
import MySQLdb
import time
import os
import os.path
import re
from decimal import *

###################### MySQLdb开始 ############################
#定义一个字典
optmap = {
'dbuser' : 'haoren',
'dbpass' : 'g6XQJLayFqDxhG4d',
'dbhost' : '172.17.112.104',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql): #定义函数
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()    #获取结果
db_cursor.close()
db_conn.close
return ret #返回结果
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return None

####################### MySQLdb结束 ###########################

###################### 文件操作类开始 #################################

#定义函数遍历当前文件夹
def traversalDir(dir):
for parent,dirnames,filenames in os.walk(rootdir): #三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames:    #输出文件夹信息
print "parent is:" + parent
print "dirname is" + dirname

for filename in filenames: #输出文件信息
print "parent is:" + parent
print "filename is:" + filename
print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
return

###################### 文件操作类结束 ##################################

#################### src_redis:源redis dst_redis:目的redis ##############################

#src_redis = redis.StrictRedis(host='localhost', port=6379, db=0)
src_redis = redis.StrictRedis(host='172.17.112.113', port=6379, db=0)
dst_redis = redis.StrictRedis(host='172.17.112.113', port=6379, db=0)

#查询相关所有的key(查询:keys*)
def getAllkeyByReqKey_src(key):
reqkey = key + "*"
ret = src_redis.keys (reqkey)
return ret

#查询当前key的类型
def getTypeByKey(key):
ret = src_redis.type (key)
return ret

#***********************zSet***********************
#zSet成员是否存在
def isMemberInzSet(key, name):
ret = src_redis.zscore (key, name)
if ret is None:
return False
else:
return True

#增加zSet
def addzSet(key, name, score, over=True):
if over == False:
   dst_redis.zincrby (key, name, int(score))
else:
   dst_redis.zadd (key,int(score), name)
return

#同步zSet
#over:覆盖
def synZSet(key, over=True):
ret = src_redis.zrevrange (key, 0, 10, withscores=True)
for list in ret:
if len(list) != 2:
print "Key(%s) 读取异常 list(%s) len(%u)" %(key, list, len(len))
else:
print "uid:%s scores:%u" %(str(list[0]), int(list[1]))
addzSet(key, list[0], list[1], over=False)
return

#检查某2个Key的数量是否相等
def checkNum4Key(srcKey, cmpKey):
ret = src_redis.zcard(srcKey)
ret1 = src_redis.zcard (cmpKey)
if ret != ret1:
print "srcKey[%s] len(%u) != cmpKey[%s] len(%u)" %(srcKey, ret, cmpKey, ret1)
return False
else:
return True

#获取所有zSet中的数据
def getzSetMember(key, withscore=False):
if withscore == False:
ret = src_redis.zrevrange (key, 0, -1, withscores=False)
return ret
else:
ret = src_redis.zrevrange(key, 0, -1, withscores=True)
return ret

#***********************zSet***********************

#***********************Set***********************
#增加Set
def addSet(key,name):
   dst_redis.sadd (key,name)
return

#同步set
def synSet(key):
ret = src_redis.smembers (key)
for list in ret:
addSet(key,list)
return
#***********************Set***********************

#***********************hash***********************
#设置Hash
def setHash(key, name, value):
ret = dst_redis.hset (key, name, value)
return

#同步hash
def synHash(key):
ret = src_redis.hgetall (key)
for name, value in ret.items():
setHash(key,name,int(value))
return

#获取Hash
def getHash(key):
ret = src_redis.hgetall (key)
return ret

#***********************hash***********************

#同步key
def synKey(key):
ret = getTypeByKey(key)
if ret == "zset":
synZSet(key)
elif ret == "set":
synSet(key)
elif ret == "hash":
synHash(key)
else:
print "当前key(%s)类型(%s),不支持同步"
return
##################################Redis结束 ##########################################

#写日志
def writeLog(txtname, contests):
strTmp = time.strftime ('% Y%m%d')
ret = os.path.exists ('/home/haoren/timingPlan/%s' %strTmp)
if ret == False:
   os.mkdir ('/home/haoren/t imingPlan/%s' %strTmp)    #建文件夹
f = file("/home/haoren/timingPlan/%s/%s" %(strTmp,txtname),'a+') #打开文件
f.write(contests)    #写内容
f.close()
return

#获取用户收费票
def getTicks(sid,curtime):
#sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= 1433124000 and OPTIME <= %s and SINGERID =%s" %(curtime,27902918)
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= %u and OPTIME <= %s and SINGERID =%s" %(int(curtime)-86400, curtime,sid)
ret = sql_select(sql) #调用函数
return ret    #返回结果

#获取用户频道昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)    #调用函数
return ret #返回结果

#获取用户当前消费
def getUserConsume(uid, curtime):
TIME_ONE_DAY = 86400
timestart = int(curtime) - TIME_ONE_DAY
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and PRESENTERID=%s" %(timestart,int(curtime),uid)
ret = sql_select(sql)    #调用函数
return ret                                  #返回结果

#分割
def splitStr(str, keystr):
sline = str.split(keystr)
return sline

#查询昨天得票总和
def getYesterdaySumTick(curtime, sid):
calctime = curtime - 86400
x = time.localtime(calctime)
strTmp = time.strftime ('%Y%m%d_',x)
strSinger = strTmp + "Singer.txt"

   #正则
r_str = "用户ID:%s" %sid
print r_str
pattern = re.compile(r_str)

f = file("/home/haoren/timingPlan/%s" %strSinger)
final = 0
for line in f.readlines():
if final == 3:
break

#先找哪一行
match = pattern.match(line.strip('\n'))
dict = {'SumTick':0,'FeeTick':0,'FreeTick':0}
if match:
   #分割空格
ret = splitStr(line.strip('\n'), ' ')
for line1 in ret:
p = re.compile(":总票数:")
#retline = line1.search(":总票数:", line1)
retline = p.search(line1)
if retline:
dict['SumTick'] = line1.strip(':总票数:')
final += 1
print dict['SumTick']

p = re.compile(":收费票:")
retline = p.search(line1)
if retline:
dict['FeeTick'] = line1.strip(':收费票:')
final += 1
print dict['FeeTick']

p = re.compile(":免费票:")
retline = p.search(line1)
if retline:
dict['FreeTick'] = line1.strip(':免费票:')
final += 1
print dict['FreeTick']
f.close()
return dict

#每日同步脚本
def recordInfo(curtime, curstage):
#获取当前时间
#strTmp = time.strftime('%Y%m%d_%H_%M_%S')
#strTmp = time.strftime('%Y%m%d_')
strTmp = time.strftime('%H_%M_%S')
strSinger = strTmp +".txt"
# reqkey = "ACTIVITY:ACTIVITYRANK:SINGERRANK:13"
# ret = getzSetMember(reqkey, withscore=True)
sum = 0

for tmp in range(0, 5):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
print tmp
ret = getzSetMember(reqkey, withscore=True)

rank = 1
for list in ret:

#获取收费票数
ticktmp = getTicks(list[0], curtime)
tick = 0
if ticktmp[0][0] != None:
tick = ticktmp[0][0] *3

nickname = getNickname(list[0])
freetick = int(list[1]) - int(tick)
if freetick < 0:
freetick = 0
#contest="用户ID:%s :总票数:%u :收费票:%u :免费票:%u " %(list[0], list[1], tick, freetick)
contest="排名%u \t用户ID:%s \t:总票数:%u \t频道昵称:%s\r\n" %(rank,list[0], list[1], nickname[0][0])
rank +=1
curSumTick = list[1]
curFeeTick = tick
curFreeTick = freetick
#contest += "今日总票数:%u 今日收费票总数:%u 今日免费票总数:%u 频道昵称:%s \r\n" %( curSumTick, curFeeTick, curFreeTick, nickname[0][0])
writeLog(strSinger,contest)
sum += list[1]
contest="当前用户总票数(%u)" %sum
writeLog(strSinger, contest) #调用函数

# strFans = strTmp + "Fans.txt"
# reqkey = "ACTIVITY:FANS:INFO"
# ret = getzSetMember(reqkey, withscore=True)
# sum = 0
# for list in ret:
# if list[1] == 0:
# continue
#
# consumeTmp = getUserConsume(list[0],curtime)
# if consumeTmp[0][0] is None:
# consume = 0
# else:
# consume = consumeTmp[0][0]
# nickname = getNickname(list[0])
# contest="UID:%s :Score:%u :活动时间累的累积消费:%u :当天消费:%u :频道昵称:%s\r\n" %(list[0],list[1], list[1]/300, consume/100, nickname[0][0])
# writeLog(strFans,contest)
# sum += list[1]
# contest="当前粉丝总票数(%u)" %sum
# writeLog(strFans, contest)
#
#
# strUser = strTmp + "User.txt"
# reqkey = "ACTIVITY:USER:INFO"
# ret = getzSetMember(reqkey, withscore=True)
# sum = 0
# for list in ret:
# if list[1] == 0:
# continue
# #过滤某用户
# if int(list[0]) == 24951487:
# continue
# if int(list[0]) == 23098877:
# continue
#
# print list[0]
# nickname = getNickname(list[0])
# contest="UID:%s:Score:%u:频道昵称:%s\r\n" %(list[0],list[1],nickname[0][0])
# writeLog(strUser,contest)
# sum += list[1]
# contest="当前活跃用户总票数(%u)" %sum
# writeLog(strUser, contest)
return

def getCurStage():
key = "SINGER:CONTEST:STAGE"
ret = getHash(key)
stage = ret['CUR:STAGE']
return stage

def main():
curTime = time.time()
curStage = getCurStage()
recordInfo(curTime, curStage)
return
main()

======================================================================================================================
#!/usr/bin/env python
#-*-coding:utf-8-*-

import redis
import sys
import MySQLdb
import time
import os
import os.path
import re
from decimal import *

##################################################
# 全局类 MySQLdb开始
##################################################
optmap = {
'dbuser' : 'haoren',
'dbpass' : 'g6XQJLayFqDxhG4d',
'dbhost' : '172.17.1.104',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()

db_cursor.close()
db_conn.close
return ret
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return None

##################################################
# 全局类 MySQLdb结束
##################################################
##################################################

##################################################
# 文件操作类开始
##################################################
#遍历当前文件夹
def traversalDir(dir):
for parent,dirnames,filenames in os.walk(rootdir): #三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames: #输出文件夹信息
print "parent is:" + parent
print "dirname is" + dirname

for filename in filenames: #输出文件信息
print "parent is:" + parent
print "filename is:" + filename
print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
return

##################################################
# 文件操作类结束
##################################################

##################################################
# 全局类 src_redis:源redis dst_redis:目的redis
##################################################
#src_redis = redis.StrictRedis(host='localhost', port=6379, db=0)
src_redis = redis.StrictRedis(host='172.17.1.113', port=6379, db=0)
dst_redis = redis.StrictRedis(host='172.17.1.113', port=6379, db=0)
#查询相关所有的key(查询:keys*)
def getAllkeyByReqKey_src(key):
reqkey = key + "*"
ret = src_redis.keys(reqkey)
return ret

#查询当前key的类型
def getTypeByKey(key):
ret = src_redis.type(key)
return ret

#***********************zSet***********************
#zSet成员是否存在
def isMemberInzSet(key, name):
ret = src_redis.zscore(key, name)
if ret is None:
return False
else:
return True

#增加zSet
def addzSet(key, name, score, over=True):
if over == False:
dst_redis.zincrby(key, name, int(score))
else:
dst_redis.zadd(key,int(score), name)
return

#同步zSet
#over:覆盖
def synZSet(key, over=True):
ret = src_redis.zrevrange(key, 0, 10, withscores=True)
for list in ret:
if len(list) != 2:
print "Key(%s) 读取异常 list(%s) len(%u)" %(key, list, len(len))
else:
print "uid:%s scores:%u" %(str(list[0]), int(list[1]))
addzSet(key, list[0], list[1], over=False)
return

#检查某2个Key的数量是否相等
def checkNum4Key(srcKey, cmpKey):
ret = src_redis.zcard(srcKey)
ret1 = src_redis.zcard(cmpKey)
if ret != ret1:
print "srcKey[%s] len(%u) != cmpKey[%s] len(%u)" %(srcKey, ret, cmpKey, ret1)
return False
else:
return True

#获取所有zSet中的数据
def getzSetMember(key, withscore=False):
if withscore == False:
ret = src_redis.zrevrange(key, 0, -1, withscores=False)
return ret
else:
ret = src_redis.zrevrange(key, 0, -1, withscores=True)
return ret

#***********************zSet***********************

#***********************Set***********************
#增加Set
def addSet(key,name):
dst_redis.sadd(key,name)
return

#同步set
def synSet(key):
ret = src_redis.smembers(key)
for list in ret:
addSet(key,list)
return
#***********************Set***********************

#***********************hash***********************
#设置Hash
def setHash(key, name, value):
ret = dst_redis.hset(key, name, value)
return

#同步hash
def synHash(key):
ret = src_redis.hgetall(key)
for name, value in ret.items():
setHash(key,name,int(value))
return

#获取Hash
def getHash(key):
ret = src_redis.hgetall(key)
return ret

#***********************hash***********************

#同步key
def synKey(key):
ret = getTypeByKey(key)
if ret == "zset":
synZSet(key)
elif ret == "set":
synSet(key)
elif ret == "hash":
synHash(key)
else:
print "当前key(%s)类型(%s),不支持同步"
return

#写日志
def writeLog(txtname, contests):
f = file("/home/haoren/timingPlan/%s" %txtname,'a+')
f.write(contests)
f.close()
return

#获取用户收费票
def getTicks(sid,curtime):
#sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= 1433124000 and OPTIME <= %s and SINGERID =%s" %(curtime,27902918)
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= %u and OPTIME <= %s and SINGERID =%s" %(int(curtime)-86400, curtime,sid)
ret = sql_select(sql)
return ret

#获取用户频道昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)
return ret

#获取用户当前消费
def getUserConsume(uid, curtime):
TIME_ONE_DAY = 86400
timestart = int(curtime) - TIME_ONE_DAY
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and PRESENTERID=%s" %(timestart,int(curtime),uid)
ret = sql_select(sql)
return ret

def splitStr(str, keystr):
sline = str.split(keystr)
return sline

#getYesTK
def getYesterdaySumTick(curtime, sid):
calctime = curtime - 86400
x = time.localtime(calctime)
strTmp = time.strftime('%Y%m%d_',x)
strSinger = strTmp + "Singer.txt"

#正则
r_str = "用户ID:%s" %sid
print r_str
pattern = re.compile(r_str)

str
f = file("/home/haoren/timingPlan/%s" %strSinger)
final = 0
for line in f.readlines():
if final == 3:
break

#先找哪一行
match = pattern.match(line.strip('\n'))
dict = {'SumTick':0,'FeeTick':0,'FreeTick':0}
if match:
#分割空格
ret = splitStr(line.strip('\n'), ' ')
for line1 in ret:
p = re.compile(":总票数:")
#retline = line1.search(":总票数:", line1)
retline = p.search(line1)
if retline:
dict['SumTick'] = line1.strip(':总票数:')
final += 1
print dict['SumTick']

p = re.compile(":收费票:")
retline = p.search(line1)
if retline:
dict['FeeTick'] = line1.strip(':收费票:')
final += 1
print dict['FeeTick']

p = re.compile(":免费票:")
retline = p.search(line1)
if retline:
dict['FreeTick'] = line1.strip(':免费票:')
final += 1
print dict['FreeTick']
f.close()
return dict

#每日同步脚本
def recordInfo(curtime, curstage):
#获取当前时间
#strTmp = time.strftime('%Y%m%d_%H_%M_%S')
#strTmp = time.strftime('%H_%M')
strTmp = time.strftime('%Y%m%d_')
strSinger = strTmp + "Singer.txt"
# reqkey = "ACTIVITY:ACTIVITYRANK:SINGERRANK:13"
# ret = getzSetMember(reqkey, withscore=True)
sum = 0

for tmp in range(0, 5):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
print tmp
ret = getzSetMember(reqkey, withscore=True)

for list in ret:

#获取收费票数
ticktmp = getTicks(list[0], curtime)
tick = 0
if ticktmp[0][0] != None:
tick = ticktmp[0][0] *3

nickname = getNickname(list[0])
freetick = int(list[1]) - int(tick)
if freetick < 0:
freetick = 0
#contest="用户ID:%s :总票数:%u :收费票:%u :免费票:%u " %(list[0], list[1], tick, freetick)
contest="用户ID:%s \t:总票数:%u \t:收费票:%u \t:免费票:%u \t频道昵称:%s \r\n" %(list[0], list[1], tick, freetick, nickname[0][0])
curSumTick = list[1]
curFeeTick = tick
curFreeTick = freetick
#contest += "今日总票数:%u 今日收费票总数:%u 今日免费票总数:%u 频道昵称:%s \r\n" %( curSumTick, curFeeTick, curFreeTick, nickname[0][0])
writeLog(strSinger,contest)
sum += list[1]
contest="当前用户总票数(%u)" %sum
writeLog(strSinger, contest)

strFans = strTmp + "Fans.txt"
reqkey = "ACTIVITY:FANS:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue

consumeTmp = getUserConsume(list[0],curtime)
if consumeTmp[0][0] is None:
consume = 0
else:
consume = consumeTmp[0][0]
nickname = getNickname(list[0])
contest="UID:%s :Score:%u :活动时间累的累积消费:%u :当天消费:%u :频道昵称:%s\r\n" %(list[0],list[1], list[1]/300, consume/100, nickname[0][0])
writeLog(strFans,contest)
sum += list[1]
contest="当前粉丝总票数(%u)" %sum
writeLog(strFans, contest)

strUser = strTmp + "User.txt"
reqkey = "ACTIVITY:USER:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue
#过滤某用户
if int(list[0]) == 24951487:
continue
if int(list[0]) == 23098877:
continue

print list[0]
nickname = getNickname(list[0])
contest="UID:%s:Score:%u:频道昵称:%s\r\n" %(list[0],list[1],nickname[0][0])
writeLog(strUser,contest)
sum += list[1]
contest="当前活跃用户总票数(%u)" %sum
writeLog(strUser, contest)
return

def getCurStage():
key = "SINGER:CONTEST:STAGE"
ret = getHash(key)
stage = ret['CUR:STAGE']
return stage

#新人换组到老用户
def main():
key_new = "ACTIVITY:GC:New:Rank:1"
key_old = "ACTIVITY:GC:Old:Rank:1"

if len(sys.argv) != 2:
print "error arg!"
return

uid = int(sys.argv[1])
ret = src_redis.zscore("ACTIVITY:GC:OLD:NEW", uid)
src_redis.zadd("ACTIVITY:GC:OLD:NEW", 1455379200, uid)
print "用户(%u) 原有是时间(%u) 修改时间:1455379200" %(uid, int(ret))

#获取新用户的分数
ret = src_redis.zscore( key_new, uid)
src_redis.zrem(key_new, uid);
print ret
src_redis.zadd(key_old, ret, uid)
main()

===============================================================================================================================================================================

#!/usr/bin/env python
#-*-coding:utf-8-*-

import redis
import sys
import MySQLdb
import time
import os
import os.path
import re
from decimal import *

##################################################
# 全局类 MySQLdb开始
##################################################
optmap = {
'dbuser' : 'haoren',
'dbpass' : 'g6XQJLayFqDxhG4d',
'dbhost' : '172.17.1.104',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()

db_cursor.close()
db_conn.close
return ret
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return None

##################################################
# 全局类 MySQLdb结束
##################################################
##################################################

##################################################
# 文件操作类开始
##################################################
#遍历当前文件夹
def traversalDir(dir):
for parent,dirnames,filenames in os.walk(rootdir): #三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames: #输出文件夹信息
print "parent is:" + parent
print "dirname is" + dirname

for filename in filenames: #输出文件信息
print "parent is:" + parent
print "filename is:" + filename
print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
return

##################################################
# 文件操作类结束
##################################################

##################################################
# 全局类 src_redis:源redis dst_redis:目的redis
##################################################
#src_redis = redis.StrictRedis(host='localhost', port=6379, db=0)
src_redis = redis.StrictRedis(host='172.17.1.113', port=6379, db=0)
dst_redis = redis.StrictRedis(host='172.17.1.113', port=6379, db=0)
#查询相关所有的key(查询:keys*)
def getAllkeyByReqKey_src(key):
reqkey = key + "*"
ret = src_redis.keys(reqkey)
return ret

#查询当前key的类型
def getTypeByKey(key):
ret = src_redis.type(key)
return ret

#***********************zSet***********************
#zSet成员是否存在
def isMemberInzSet(key, name):
ret = src_redis.zscore(key, name)
if ret is None:
return False
else:
return True

#增加zSet
def addzSet(key, name, score, over=True):
if over == False:
dst_redis.zincrby(key, name, int(score))
else:
dst_redis.zadd(key,int(score), name)
return

#同步zSet
#over:覆盖
def synZSet(key, over=True):
ret = src_redis.zrevrange(key, 0, 10, withscores=True)
for list in ret:
if len(list) != 2:
print "Key(%s) 读取异常 list(%s) len(%u)" %(key, list, len(len))
else:
print "uid:%s scores:%u" %(str(list[0]), int(list[1]))
addzSet(key, list[0], list[1], over=False)
return

#检查某2个Key的数量是否相等
def checkNum4Key(srcKey, cmpKey):
ret = src_redis.zcard(srcKey)
ret1 = src_redis.zcard(cmpKey)
if ret != ret1:
print "srcKey[%s] len(%u) != cmpKey[%s] len(%u)" %(srcKey, ret, cmpKey, ret1)
return False
else:
return True

#获取所有zSet中的数据
def getzSetMember(key, withscore=False):
if withscore == False:
ret = src_redis.zrevrange(key, 0, -1, withscores=False)
return ret
else:
ret = src_redis.zrevrange(key, 0, -1, withscores=True)
return ret

#***********************zSet***********************

#***********************Set***********************
#增加Set
def addSet(key,name):
dst_redis.sadd(key,name)
return

#同步set
def synSet(key):
ret = src_redis.smembers(key)
for list in ret:
addSet(key,list)
return
#***********************Set***********************

#***********************hash***********************
#设置Hash
def setHash(key, name, value):
ret = dst_redis.hset(key, name, value)
return

#同步hash
def synHash(key):
ret = src_redis.hgetall(key)
for name, value in ret.items():
setHash(key,name,int(value))
return

#获取Hash
def getHash(key):
ret = src_redis.hgetall(key)
return ret

#***********************hash***********************

#同步key
def synKey(key):
ret = getTypeByKey(key)
if ret == "zset":
synZSet(key)
elif ret == "set":
synSet(key)
elif ret == "hash":
synHash(key)
else:
print "当前key(%s)类型(%s),不支持同步"
return

#写日志
def writeLog(txtname, contests):
f = file("/home/haoren/timingPlan/ceshi/%s" %txtname,'a+')
f.write(contests)
f.close()
return

#获取用户收费票
def getTicks(sid,curtime):
#sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= 1433124000 and OPTIME <= %s and SINGERID =%s" %(curtime,27902918)
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= %u and OPTIME <= %s and SINGERID =%s" %(int(curtime)-86400, curtime,sid)
ret = sql_select(sql)
return ret

#获取用户频道昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)
return ret

#获取用户当前消费
def getUserConsume(uid, curtime):
TIME_ONE_DAY = 86400
timestart = int(curtime) - TIME_ONE_DAY
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and PRESENTERID=%s" %(timestart,int(curtime),uid)
ret = sql_select(sql)
return ret

def splitStr(str, keystr):
sline = str.split(keystr)
return sline

#getYesTK
def getYesterdaySumTick(curtime, sid):
calctime = curtime - 86400
x = time.localtime(calctime)
strTmp = time.strftime('%Y%m%d_',x)
strSinger = strTmp + "Singer.txt"

#正则
r_str = "用户ID:%s" %sid
print r_str
pattern = re.compile(r_str)

f = file("/home/haoren/timingPlan/%s" %strSinger)
final = 0
for line in f.readlines():
if final == 3:
break

#先找哪一行
match = pattern.match(line.strip('\n'))
dict = {'SumTick':0,'FeeTick':0,'FreeTick':0}
if match:
#分割空格
ret = splitStr(line.strip('\n'), ' ')
for line1 in ret:
p = re.compile(":总票数:")
#retline = line1.search(":总票数:", line1)
retline = p.search(line1)
if retline:
dict['SumTick'] = line1.strip(':总票数:')
final += 1
print dict['SumTick']

p = re.compile(":收费票:")
retline = p.search(line1)
if retline:
dict['FeeTick'] = line1.strip(':收费票:')
final += 1
print dict['FeeTick']

p = re.compile(":免费票:")
retline = p.search(line1)
if retline:
dict['FreeTick'] = line1.strip(':免费票:')
final += 1
print dict['FreeTick']
f.close()
return dict

#每日同步脚本
def recordInfo(curtime, curstage):
#获取当前时间
#strTmp = time.strftime('%Y%m%d_%H_%M_%S')
strTmp = time.strftime('%Y%m%d_')
strSinger = strTmp + "Singer.txt"
# reqkey = "ACTIVITY:ACTIVITYRANK:SINGERRANK:13"
# ret = getzSetMember(reqkey, withscore=True)
sum = 0

for tmp in range(0, 5):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
print tmp
ret = getzSetMember(reqkey, withscore=True)

for list in ret:

#获取收费票数
ticktmp = getTicks(list[0], curtime)
tick = 0
if ticktmp[0][0] != None:
tick = ticktmp[0][0] *3

nickname = getNickname(list[0])
freetick = int(list[1]) - int(tick)
if freetick < 0:
freetick = 0
#contest="用户ID:%s :总票数:%u :收费票:%u :免费票:%u " %(list[0], list[1], tick, freetick)
contest="用户ID:%s \t:总票数:%u \t:收费票:%u \t:免费票:%u \t频道昵称:%s\r\n" %(list[0], list[1], tick, freetick, nickname[0][0])
curSumTick = list[1]
curFeeTick = tick
curFreeTick = freetick
#contest += "今日总票数:%u 今日收费票总数:%u 今日免费票总数:%u 频道昵称:%s \r\n" %( curSumTick, curFeeTick, curFreeTick, nickname[0][0])
writeLog(strSinger,contest)
sum += list[1]
contest="当前用户总票数(%u)" %sum
writeLog(strSinger, contest)

strFans = strTmp + "Fans.txt"
reqkey = "ACTIVITY:FANS:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue

consumeTmp = getUserConsume(list[0],curtime)
if consumeTmp[0][0] is None:
consume = 0
else:
consume = consumeTmp[0][0]
nickname = getNickname(list[0])
contest="UID:%s :Score:%u :活动时间累的累积消费:%u :当天消费:%u :频道昵称:%s\r\n" %(list[0],list[1], list[1]/300, consume/100, nickname[0][0])
writeLog(strFans,contest)
sum += list[1]
contest="当前粉丝总票数(%u)" %sum
writeLog(strFans, contest)

strUser = strTmp + "User.txt"
reqkey = "ACTIVITY:USER:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue
#过滤某用户
if int(list[0]) == 24951487:
continue
if int(list[0]) == 23098877:
continue

print list[0]
nickname = getNickname(list[0])
contest="UID:%s:Score:%u:频道昵称:%s\r\n" %(list[0],list[1],nickname[0][0])
writeLog(strUser,contest)
sum += list[1]
contest="当前活跃用户总票数(%u)" %sum
writeLog(strUser, contest)
return

def getCurStage():
key = "SINGER:CONTEST:STAGE"
ret = getHash(key)
stage = ret['CUR:STAGE']
return stage

def getUserConsume(uid, time1, time2 = 0):
if time2 != 0:
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and SINGERID=%s" %(int(time1),int(time2),uid)
else:
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and SINGERID=%s" %(int(time1),uid)
ret = sql_select(sql)
return ret

#call 获取用户的免费票
def getFreeTick(sid):

cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(30)\\\" /log/activityserver.log.150611-1[6-9] | grep \\\"%u\\\" \"|awk -F '[()]' 'BEGIN{sum=0}{sum+=$10}END {print sum}'" %sid
output = os.popen(str(cmd_ssh))
ret_ssh = output.read()

cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(30)\\\" /log/activityserver.log.150611-2[0-1] | grep \\\"%u\\\" \"|awk -F '[()]' 'BEGIN{sum=0}{sum+=$10}END {print sum}'" %sid
output = os.popen(str(cmd_ssh))
ret_ssh1 = output.read()

return int(ret_ssh) + int(ret_ssh1)

#call 获取用户的免费票
def getOfflineTick(sid):

cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(31)\\\" /log/activityserver.log.150611-1[6-9] |grep \\\"%u\\\" \"|awk -F '[()]' 'BEGIN{sum=0}{sum+=$10}END {print sum}'" %sid
output = os.popen(str(cmd_ssh))
ret_ssh = output.read()
return int(ret_ssh)

def synTick(curstage):
for tmp in range(0, 4):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
print reqkey

ret = getzSetMember(reqkey, withscore=True)

for list in ret:
#获取收费票数(2015-6-11 16:00:00)
timestart = 1434009600
timeend = 0
ret1 = getUserConsume(list[0],timestart, 0)

if ret1[0][0] is None:
feeTick = 0
else:
feeTick = int(ret1[0][0])*3

#获取用户免票
freeTick = getFreeTick( int(list[0]) )

#获取离线票
offlineTick = getOfflineTick( int(list[0]))
if offlineTick+freeTick+feeTick != list[1]:
print "用户id(%u) \t收费票(%u) \t免费票(%u) \t离线票(%u) \t票总数(%u) \tredis票(%u)"\
%( int(list[0]),freeTick,feeTick,offlineTick, offlineTick+freeTick+feeTick,list[1])

addzSet(reqkey, list[0], freeTick+feeTick, over=True)
return

#同步票脚本修改点：
#1. 收费票时间
#2. 免费票日志时间
def main():
key="ActivityServer:CB:Daily:Performer:2016:06:06"
key1="ActivityServer:CB:Totoal:Performer"
ret = getzSetMember(key,True)
src_redis.delete(key1)
for i in ret:
#print i[0],i[1]
addzSet(key1, int(i[0]), int(i[1]), True)

return
main()

=====================================================================================================================================================================
#!/usr/bin/env python

import redis
import sys
import os
import time

def_rds = redis.StrictRedis(host='172.17.1.119',port='6379',db=0)

def printKey(key, curtime):
ret = def_rds.zrevrange(key, 0, -1, withscores=True)
curtime = float(curtime)
ltime = time.localtime(curtime)
stime = time.strftime("%Y%m%d %H:%M:%S", ltime)

pack = {}
for lst in ret:
ret = pack.get(lst[0])
print stime, lst[0], lst[1]

def main():
uid = sys.argv[1]
ret = def_rds.smembers("%s:package:set" %uid)
print ret
print ret
for lst in ret:
rdsKey = "%s:%s" %(uid, lst)
printKey( rdsKey, lst)
main()

===================================================================================================================================================================
#!/usr/bin/env python
#-*-coding:utf-8-*-

import redis
import sys
import MySQLdb
import time
import os
import os.path
import re
from decimal import *

##################################################
# 全局类 MySQLdb开始
##################################################
optmap = {
'dbuser' : 'haoren',
'dbpass' : 'g6XQJLayFqDxhG4d',
'dbhost' : '172.17.1.125',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()

db_cursor.close()
db_conn.close
return ret
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return None

##################################################
# 全局类 MySQLdb结束
##################################################
##################################################

##################################################
# 文件操作类开始
##################################################
#遍历当前文件夹
def traversalDir(dir):
for parent,dirnames,filenames in os.walk(rootdir): #三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames: #输出文件夹信息
print "parent is:" + parent
print "dirname is" + dirname

for filename in filenames: #输出文件信息
print "parent is:" + parent
print "filename is:" + filename
print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
return

##################################################
# 文件操作类结束
##################################################

##################################################
# 全局类 src_redis:源redis dst_redis:目的redis
##################################################
#src_redis = redis.StrictRedis(host='localhost', port=6379, db=0)
src_redis = redis.StrictRedis(host='172.17.1.113', port=6379, db=0)
dst_redis = redis.StrictRedis(host='172.17.1.113', port=6379, db=0)
#查询相关所有的key(查询:keys*)
def getAllkeyByReqKey_src(key):
reqkey = key + "*"
ret = src_redis.keys(reqkey)
return ret

#查询当前key的类型
def getTypeByKey(key):
ret = src_redis.type(key)
return ret

#***********************zSet***********************
#zSet成员是否存在
def isMemberInzSet(key, name):
ret = src_redis.zscore(key, name)
if ret is None:
return False
else:
return True

#增加zSet
def addzSet(key, name, score, over=True):
if over == False:
dst_redis.zincrby(key, name, int(score))
else:
dst_redis.zadd(key,int(score), name)
return

#同步zSet
#over:覆盖
def synZSet(key, over=True):
ret = src_redis.zrevrange(key, 0, 10, withscores=True)
for list in ret:
if len(list) != 2:
print "Key(%s) 读取异常 list(%s) len(%u)" %(key, list, len(len))
else:
print "uid:%s scores:%u" %(str(list[0]), int(list[1]))
addzSet(key, list[0], list[1], over=False)
return

#检查某2个Key的数量是否相等
def checkNum4Key(srcKey, cmpKey):
ret = src_redis.zcard(srcKey)
ret1 = src_redis.zcard(cmpKey)
if ret != ret1:
print "srcKey[%s] len(%u) != cmpKey[%s] len(%u)" %(srcKey, ret, cmpKey, ret1)
return False
else:
return True

#获取所有zSet中的数据
def getzSetMember(key, withscore=False):
if withscore == False:
ret = src_redis.zrevrange(key, 0, -1, withscores=False)
return ret
else:
ret = src_redis.zrevrange(key, 0, -1, withscores=True)
return ret

#***********************zSet***********************

#***********************Set***********************
#增加Set
def addSet(key,name):
dst_redis.sadd(key,name)
return

#同步set
def synSet(key):
ret = src_redis.smembers(key)
for list in ret:
addSet(key,list)
return
#***********************Set***********************

#***********************hash***********************
#设置Hash
def setHash(key, name, value):
ret = dst_redis.hset(key, name, value)
return

#同步hash
def synHash(key):
ret = src_redis.hgetall(key)
for name, value in ret.items():
setHash(key,name,int(value))
return

#获取Hash
def getHash(key):
ret = src_redis.hgetall(key)
return ret

#***********************hash***********************

#同步key
def synKey(key):
ret = getTypeByKey(key)
if ret == "zset":
synZSet(key)
elif ret == "set":
synSet(key)
elif ret == "hash":
synHash(key)
else:
print "当前key(%s)类型(%s),不支持同步"
return

#写日志
def writeLog(txtname, contests):
f = file("/home/haoren/timingPlan/ceshi/%s" %txtname,'a')
f.write(contests)
f.close()
return

#获取用户收费票
def getTicks(sid,curtime):
#sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= 1433124000 and OPTIME <= %s and SINGERID =%s" %(curtime,27902918)
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= %u and OPTIME <= %s and SINGERID =%s" %(int(curtime)-86400, curtime,sid)
ret = sql_select(sql)
return ret

#获取用户频道昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)
return ret

#获取用户当前消费
def getUserConsume(uid, curtime):
TIME_ONE_DAY = 86400
timestart = int(curtime) - TIME_ONE_DAY
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and PRESENTERID=%s" %(timestart,int(curtime),uid)
ret = sql_select(sql)
return ret

def splitStr(str, keystr):
sline = str.split(keystr)
return sline

#getYesTK
def getYesterdaySumTick(curtime, sid):
calctime = curtime - 86400
x = time.localtime(calctime)
strTmp = time.strftime('%Y%m%d_',x)
strSinger = strTmp + "Singer.txt"

#正则
r_str = "用户ID:%s" %sid
print r_str
pattern = re.compile(r_str)

f = file("/home/haoren/timingPlan/%s" %strSinger)
final = 0
for line in f.readlines():
if final == 3:
break

#先找哪一行
match = pattern.match(line.strip('\n'))
dict = {'SumTick':0,'FeeTick':0,'FreeTick':0}
if match:
#分割空格
ret = splitStr(line.strip('\n'), ' ')
for line1 in ret:
p = re.compile(":总票数:")
#retline = line1.search(":总票数:", line1)
retline = p.search(line1)
if retline:
dict['SumTick'] = line1.strip(':总票数:')
final += 1
print dict['SumTick']

p = re.compile(":收费票:")
retline = p.search(line1)
if retline:
dict['FeeTick'] = line1.strip(':收费票:')
final += 1
print dict['FeeTick']

p = re.compile(":免费票:")
retline = p.search(line1)
if retline:
dict['FreeTick'] = line1.strip(':免费票:')
final += 1
print dict['FreeTick']
f.close()
return dict

#每日同步脚本
def recordInfo(curtime, curstage):
#获取当前时间
#strTmp = time.strftime('%Y%m%d_%H_%M_%S')
strTmp = time.strftime('%Y%m%d_')
strSinger = strTmp + "Singer.txt"
# reqkey = "ACTIVITY:ACTIVITYRANK:SINGERRANK:13"
# ret = getzSetMember(reqkey, withscore=True)
sum = 0

for tmp in range(0, 5):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
print tmp
ret = getzSetMember(reqkey, withscore=True)

for list in ret:

#获取收费票数
ticktmp = getTicks(list[0], curtime)
tick = 0
if ticktmp[0][0] != None:
tick = ticktmp[0][0] *3

nickname = getNickname(list[0])
freetick = int(list[1]) - int(tick)
if freetick < 0:
freetick = 0
#contest="用户ID:%s :总票数:%u :收费票:%u :免费票:%u " %(list[0], list[1], tick, freetick)
contest="用户ID:%s \t:总票数:%u \t:收费票:%u \t:免费票:%u \t频道昵称:%s\r\n" %(list[0], list[1], tick, freetick, nickname[0][0])
curSumTick = list[1]
curFeeTick = tick
curFreeTick = freetick
#contest += "今日总票数:%u 今日收费票总数:%u 今日免费票总数:%u 频道昵称:%s \r\n" %( curSumTick, curFeeTick, curFreeTick, nickname[0][0])
writeLog(strSinger,contest)
sum += list[1]
contest="当前用户总票数(%u)" %sum
writeLog(strSinger, contest)

strFans = strTmp + "Fans.txt"
reqkey = "ACTIVITY:FANS:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue

consumeTmp = getUserConsume(list[0],curtime)
if consumeTmp[0][0] is None:
consume = 0
else:
consume = consumeTmp[0][0]
nickname = getNickname(list[0])
contest="UID:%s :Score:%u :活动时间累的累积消费:%u :当天消费:%u :频道昵称:%s\r\n" %(list[0],list[1], list[1]/300, consume/100, nickname[0][0])
writeLog(strFans,contest)
sum += list[1]
contest="当前粉丝总票数(%u)" %sum
writeLog(strFans, contest)

strUser = strTmp + "User.txt"
reqkey = "ACTIVITY:USER:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue
#过滤某用户
if int(list[0]) == 24951487:
continue
if int(list[0]) == 23098877:
continue

print list[0]
nickname = getNickname(list[0])
contest="UID:%s:Score:%u:频道昵称:%s\r\n" %(list[0],list[1],nickname[0][0])
writeLog(strUser,contest)
sum += list[1]
contest="当前活跃用户总票数(%u)" %sum
writeLog(strUser, contest)
return

def getCurStage():
key = "SINGER:CONTEST:STAGE"
ret = getHash(key)
stage = ret['CUR:STAGE']
return stage

def getUserConsume(uid, time1, time2 = 0):
if time2 != 0:
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and SINGERID=%s" %(int(time1),int(time2),uid)
else:
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and SINGERID=%s" %(int(time1),uid)
ret = sql_select(sql)
return ret

def synTick(timestart, timeend):
return

#call 获取用户的免费票
def getFreeTick(sid):

cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(30)\\\" /log/activityserver.log.150611-1[6-8] |grep \\\"%u\\\" \"|awk -F '[()]' 'BEGIN{sum=0}{sum+=$10}END {print sum}'" %sid
output = os.popen(str(cmd_ssh))
ret_ssh = output.read()
return int(ret_ssh)

def SumTickByStage(curstage, sid):

for tmp in range(0, 4):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
ret = getzSetMember(reqkey, withscore=True)
for list in ret:
if sid == list[0]:
return list[1]
else:
continue
return

def getDayByTime(curtime):
tmp = time.localtime(curtime)
ret = time.strftime('%m%d',tmp)
return ret

def exeSshCmd(cmd_ssh):
output = os.popen(str(cmd_ssh))
ret_ssh = output.read()
#print ret_ssh
return ret_ssh

def getMaxStr(str1, str2):
if len(str1) > len(str2):
return str1
elif len(str1) < len(str2):
return str2
else:
ret = cmp(str1, str2)
if ret > 1:
return str1
else:
return str2
return str2

def getSingerLevel(filename,sid):
cmd_ssh = "ssh 172.17.1.125 \" grep -rn 物品统计 /home/haoren/logdir/15%s_67/billserver.log*\
|grep \\\"用户(%s)\\\" \" | tail -1 |awk -F'[()]' '{print $14}'" %(filename, str(sid))
ret = exeSshCmd(cmd_ssh)
ret = ret.strip('\n')
ret = ret.strip(' ')
return int(ret)

def getCurGift(str):
ret = str.split('/')
return ret[0]

def reqTask(filename):
log_addi_list =['2', '29', '76']
dict = {}

for iplist in log_addi_list:
#cmd_ssh = "ssh 172.17.1.125 \"grep -rn 用户任务 /home/haoren/logdir/150501_2/vchannelserver.log* \
# |grep 完成物品任务 |grep 49132813 \"|awk -F '[()]' '{print $2, $4}'"

cmd_ssh = "ssh 172.17.1.125 \"grep -rn 用户任务 /home/haoren/logdir/15%s_%s/vchannelserver.log* \
|grep 完成物品任务 \"|awk -F '[()]' '{print $2, $4}'" %(filename, iplist)

ret = exeSshCmd(cmd_ssh)

#解析字符串分割成2个
ret_list = ret.split()
len_list = len(ret_list)

if len_list == 0:
continue

for list in range (0, len_list/2):
curId = ret_list[list*2]
curNum = ret_list[list*2+1]

if len(dict) == 0:
dict[curId] = curNum
else:
for id, socre in dict.items():
if int(id) == int(curId):
ret = getMaxStr( dict[id], curNum )
dict[id] = ret
continue
#这里是没有查询到
dict[curId] = curNum

#当前字典写入文本中
logfile = "singerTask/%s.txt" %filename

for id, scores in dict.items():
nickname = getNickname(id)
#level = getSingerLevel(filename,id)
curNum = getCurGift(scores)
contest = "%s,%s,%s\r\n" %(id, scores, nickname[0][0])
writeLog(logfile, contest)
return

def getSocre(day):
key = "ACTIVITY:SINGER:CHARM:2015:%u" %day
ret_list = getzSetMember(key,withscore=True)
return ret_list

def count(idx):
keyname = "SC5:TOTAL:GROUP:INFO:%u" %(int(idx))
ret = dst_redis.zrevrange(keyname, 0, -1, True)

for lst in ret:
print "UID:%u,SCORE:%u" %(int(lst[0]), int(lst[1]))

return

#统计每一个总榜的用户
def main():
#统计最近7天的魅力值
for idx in range (0, 5):
count(idx)
main()

==================================================================================================================================================================================================

#!/usr/bin/env python
#-*-coding:utf-8-*-

import redis
import sys
import MySQLdb
import time
import os
import os.path
import re
from decimal import *

##################################################
# 全局类 MySQLdb开始
##################################################
optmap = {
'dbuser' : 'haoren',
'dbpass' : 'g6XQJLayFqDxhG4d',
'dbhost' : '172.17.1.104',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()

db_cursor.close()
db_conn.close
return ret
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return None

##################################################
# 全局类 MySQLdb结束
##################################################
##################################################

##################################################
# 文件操作类开始
##################################################
#遍历当前文件夹
def traversalDir(dir):
for parent,dirnames,filenames in os.walk(rootdir): #三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames: #输出文件夹信息
print "parent is:" + parent
print "dirname is" + dirname

for filename in filenames: #输出文件信息
print "parent is:" + parent
print "filename is:" + filename
print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
return

##################################################
# 文件操作类结束
##################################################

##################################################
# 全局类 src_redis:源redis dst_redis:目的redis
##################################################
#src_redis = redis.StrictRedis(host='localhost', port=6379, db=0)
src_redis = redis.StrictRedis(host='172.17.1.119', port=6379, db=0)
dst_redis = redis.StrictRedis(host='172.17.1.119', port=6379, db=0)
#查询相关所有的key(查询:keys*)
def getAllkeyByReqKey_src(key):
reqkey = key + "*"
ret = src_redis.keys(reqkey)
return ret

#查询当前key的类型
def getTypeByKey(key):
ret = src_redis.type(key)
return ret

def r_smembers(key):
ret = src_redis.smembers( str(key))
return ret

def r_zscores(key, name):
ret = src_redis.zscore( str(key), str(name))
return ret

#***********************zSet***********************
#zSet成员是否存在
def isMemberInzSet(key, name):
ret = src_redis.zscore(key, name)
if ret is None:
return False
else:
return True

#增加zSet
def addzSet(key, name, score, over=True):
if over == False:
dst_redis.zincrby(key, name, int(score))
else:
dst_redis.zadd(key,int(score), name)
return

#同步zSet
#over:覆盖
def synZSet(key, over=True):
ret = src_redis.zrevrange(key, 0, 10, withscores=True)
for list in ret:
if len(list) != 2:
print "Key(%s) 读取异常 list(%s) len(%u)" %(key, list, len(len))
else:
print "uid:%s scores:%u" %(str(list[0]), int(list[1]))
addzSet(key, list[0], list[1], over=False)
return

#检查某2个Key的数量是否相等
def checkNum4Key(srcKey, cmpKey):
ret = src_redis.zcard(srcKey)
ret1 = src_redis.zcard(cmpKey)
if ret != ret1:
print "srcKey[%s] len(%u) != cmpKey[%s] len(%u)" %(srcKey, ret, cmpKey, ret1)
return False
else:
return True

#获取所有zSet中的数据
def getzSetMember(key, withscore=False):
if withscore == False:
ret = src_redis.zrevrange(key, 0, -1, withscores=False)
return ret
else:
ret = src_redis.zrevrange(key, 0, -1, withscores=True)
return ret

#***********************zSet***********************

#***********************Set***********************
#增加Set
def addSet(key,name):
dst_redis.sadd(key,name)
return

#同步set
def synSet(key):
ret = src_redis.smembers(key)
for list in ret:
addSet(key,list)
return
#***********************Set***********************

#***********************hash***********************
#设置Hash
def setHash(key, name, value):
ret = dst_redis.hset(key, name, value)
return

#同步hash
def synHash(key):
ret = src_redis.hgetall(key)
for name, value in ret.items():
setHash(key,name,int(value))
return

#获取Hash
def getHash(key):
ret = src_redis.hgetall(key)
return ret

#***********************hash***********************

#同步key
def synKey(key):
ret = getTypeByKey(key)
if ret == "zset":
synZSet(key)
elif ret == "set":
synSet(key)
elif ret == "hash":
synHash(key)
else:
print "当前key(%s)类型(%s),不支持同步"
return

#写日志
def writeLog(txtname, contests):
f = file("/home/haoren/timingPlan/ceshi/%s" %txtname,'a+')
f.write(contests)
f.close()
return

#获取用户收费票
def getTicks(sid,curtime):
#sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= 1433124000 and OPTIME <= %s and SINGERID =%s" %(curtime,27902918)
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >= %u and OPTIME <= %s and SINGERID =%s" %(int(curtime)-86400, curtime,sid)
ret = sql_select(sql)
return ret

#获取用户频道昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)
return ret

#获取用户当前消费
def getUserConsume(uid, curtime):
TIME_ONE_DAY = 86400
timestart = int(curtime) - TIME_ONE_DAY
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and PRESENTERID=%s" %(timestart,int(curtime),uid)
ret = sql_select(sql)
return ret

def splitStr(str, keystr):
sline = str.split(keystr)
return sline

#getYesTK
def getYesterdaySumTick(curtime, sid):
calctime = curtime - 86400
x = time.localtime(calctime)
strTmp = time.strftime('%Y%m%d_',x)
strSinger = strTmp + "Singer.txt"

#正则
r_str = "用户ID:%s" %sid
print r_str
pattern = re.compile(r_str)

f = file("/home/haoren/timingPlan/%s" %strSinger)
final = 0
for line in f.readlines():
if final == 3:
break

#先找哪一行
match = pattern.match(line.strip('\n'))
dict = {'SumTick':0,'FeeTick':0,'FreeTick':0}
if match:
#分割空格
ret = splitStr(line.strip('\n'), ' ')
for line1 in ret:
p = re.compile(":总票数:")
#retline = line1.search(":总票数:", line1)
retline = p.search(line1)
if retline:
dict['SumTick'] = line1.strip(':总票数:')
final += 1
print dict['SumTick']

p = re.compile(":收费票:")
retline = p.search(line1)
if retline:
dict['FeeTick'] = line1.strip(':收费票:')
final += 1
print dict['FeeTick']

p = re.compile(":免费票:")
retline = p.search(line1)
if retline:
dict['FreeTick'] = line1.strip(':免费票:')
final += 1
print dict['FreeTick']
f.close()
return dict

#每日同步脚本
def recordInfo(curtime, curstage):
#获取当前时间
#strTmp = time.strftime('%Y%m%d_%H_%M_%S')
strTmp = time.strftime('%Y%m%d_')
strSinger = strTmp + "Singer.txt"
# reqkey = "ACTIVITY:ACTIVITYRANK:SINGERRANK:13"
# ret = getzSetMember(reqkey, withscore=True)
sum = 0

for tmp in range(0, 5):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
print tmp
ret = getzSetMember(reqkey, withscore=True)

for list in ret:

#获取收费票数
ticktmp = getTicks(list[0], curtime)
tick = 0
if ticktmp[0][0] != None:
tick = ticktmp[0][0] *3

nickname = getNickname(list[0])
freetick = int(list[1]) - int(tick)
if freetick < 0:
freetick = 0
#contest="用户ID:%s :总票数:%u :收费票:%u :免费票:%u " %(list[0], list[1], tick, freetick)
contest="用户ID:%s \t:总票数:%u \t:收费票:%u \t:免费票:%u \t频道昵称:%s\r\n" %(list[0], list[1], tick, freetick, nickname[0][0])
curSumTick = list[1]
curFeeTick = tick
curFreeTick = freetick
#contest += "今日总票数:%u 今日收费票总数:%u 今日免费票总数:%u 频道昵称:%s \r\n" %( curSumTick, curFeeTick, curFreeTick, nickname[0][0])
writeLog(strSinger,contest)
sum += list[1]
contest="当前用户总票数(%u)" %sum
writeLog(strSinger, contest)

strFans = strTmp + "Fans.txt"
reqkey = "ACTIVITY:FANS:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue

consumeTmp = getUserConsume(list[0],curtime)
if consumeTmp[0][0] is None:
consume = 0
else:
consume = consumeTmp[0][0]
nickname = getNickname(list[0])
contest="UID:%s :Score:%u :活动时间累的累积消费:%u :当天消费:%u :频道昵称:%s\r\n" %(list[0],list[1], list[1]/300, consume/100, nickname[0][0])
writeLog(strFans,contest)
sum += list[1]
contest="当前粉丝总票数(%u)" %sum
writeLog(strFans, contest)

strUser = strTmp + "User.txt"
reqkey = "ACTIVITY:USER:INFO"
ret = getzSetMember(reqkey, withscore=True)
sum = 0
for list in ret:
if list[1] == 0:
continue
#过滤某用户
if int(list[0]) == 24951487:
continue
if int(list[0]) == 23098877:
continue

print list[0]
nickname = getNickname(list[0])
contest="UID:%s:Score:%u:频道昵称:%s\r\n" %(list[0],list[1],nickname[0][0])
writeLog(strUser,contest)
sum += list[1]
contest="当前活跃用户总票数(%u)" %sum
writeLog(strUser, contest)
return

def getCurStage():
key = "SINGER:CONTEST:STAGE"
ret = getHash(key)
stage = ret['CUR:STAGE']
return stage

def getUserConsume(uid, time1, time2 = 0):
if time2 != 0:
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and OPTIME <= %u and SINGERID=%s" %(int(time1),int(time2),uid)
else:
sql = "select sum(CONSUMECOIN) from ITEMCONSUMERECORD where OPTIME >=%u and SINGERID=%s" %(int(time1),uid)
ret = sql_select(sql)
return ret

def synTick(timestart, timeend):
return

#call 获取用户的免费票
def getFreeTick(sid):

cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(30)\\\" /log/activityserver.log.150611-1[6-8] |grep \\\"%u\\\" \"|awk -F '[()]' 'BEGIN{sum=0}{sum+=$10}END {print sum}'" %sid
output = os.popen(str(cmd_ssh))
ret_ssh = output.read()
return int(ret_ssh)

def SumTickByStage(curstage, sid):

for tmp in range(0, 4):
reqkey = "SINGER:GROUP:INFO:%u:%u" %(int(curstage), tmp)
ret = getzSetMember(reqkey, withscore=True)
for list in ret:
if sid == list[0]:
return list[1]
else:
continue
return

#同步票脚本修改点：
#1. 收费票时间
#2. 免费票日志时间
def main():
if len(sys.argv) != 3:
print "Paramer is error!"
exit()

id = int( sys.argv[1] )
itemid = int( sys.argv[2])
print id, itemid

sql_rd = "%u:package:set" %(id)
ret = r_smembers(sql_rd)
scores = 0
for lst in ret:
sql_rd1 = "%u:%u" %( id, int(lst))
ret_score = r_zscores( str(sql_rd1), itemid )
if ret_score == None:
continue
else:
scores += ret_score
print ret_score

print "Sum 用户(%u) 拥有(%u) (%u)个" %(id, itemid, scores)
return
main()

================================================================================================================================================================================================
#!/usr/bin/env python
#-*-coding:utf-8-*-

import redis
import sys
import MySQLdb
import datetime
import time
import os
import os.path

#-------------------------
#name: dbitem_to_fengshenbang_redis.py
#Purpose: 导出物品记录到封神榜Redis
#Author:
#--------------------------

##################################################
# 全局类 MySQLdb开始
##################################################
optmap = {
'dbuser' : 'haoren',
'dbpass' : 'g6XQJLayFqDxhG4d',
'dbhost' : '172.17.1.125',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()

db_cursor.close()
db_conn.close
return ret
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return ''

global gdb_conn
global gdb_cursor
def mysql_connect():
try:
global gdb_conn
global gdb_cursor
gdb_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
gdb_cursor=gdb_conn.cursor()
#gdb_conn.query("use %s"%optmap['dbname'])
#print "Mysql Connect Success:%s %s=%s %s \r\n" %(optmap['dbhost'], optmap['dbuser'], optmap['dbpass'], optmap['dbname'])
return
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return

def mysql_close():
global gdb_conn
global gdb_cursor
gdb_cursor.close()
gdb_conn.close()
return

##################################################
# 全局类 MySQLdb结束
##################################################

##################################################
# 全局类 src_redis:源redis dst_redis:目的redis
##################################################
#src_redis = redis.StrictRedis(host='localhost', port=6379, db=0)
src_redis = redis.StrictRedis(host='172.17.1.111', port=6379, db=0)
dst_redis = redis.StrictRedis(host='172.17.1.111', port=6379, db=0)
#查询相关所有的key(查询:keys*)
def getAllkeyByReqKey_src(key):
reqkey = key + "*"
ret = src_redis.keys(reqkey)
return ret

#查询当前key的类型
def getTypeByKey(key):
ret = src_redis.type(key)
return ret

#删除Key
def deleteKey(key):
ret = dst_redis.delete(key)
return ret

#zSet成员是否存在
def isMemberInzSet(key, name):
ret = src_redis.zscore(key, name)
if ret is None:
return False
else:
return True

#增加zSet
def addzSet(key, name, score, over=True):
if over == False:
dst_redis.zincrby(key, name, int(score))
else:
dst_redis.zadd(key,int(score), name )
return

#同步zSet
#over:覆盖
def synZSet(key, over=True):
ret = src_redis.zrevrange(key, 0, 10, withscores=True)
for list in ret:
if len(list) != 2:
print "Key(%s) 读取异常 list(%s) len(%u)" %(key, list, len(len))
else:
print "uid:%s scores:%u" %(str(list[0]), int(list[1]))
addzSet(key, list[0], list[1], over=False)
return

#增加Set
def addSet(key,name):
dst_redis.sadd(key,name)
return

#同步set
def synSet(key):
ret = src_redis.smembers(key)
for list in ret:
print list,
addSet(key,list)
return

#设置Hash
def setHash(key, name, value):
ret = dst_redis.hset(key, name, value)
return

#同步hash
def synHash(key):
ret = src_redis.hgetall(key)
for name, value in ret.items():
setHash(key,name,int(value))
return

#同步key
def synKey(key):
ret = getTypeByKey(key)
if ret == "zset":
synZSet(key)
elif ret == "set":
synSet(key)
elif ret == "hash":
synHash(key)
else:
print "当前key(%s)类型(%s),不支持同步"
return

##################################################
# 时间操作开始
##################################################
startdate = datetime.date(2015, 7, 18)
datelen = 31
def fromDay(func):
func(startdate)

def fromDayToDay(func):
delta = datetime.timedelta(days=1)
for i in range(0,datelen):
startday = startdate + delta * i
endday = startdate + delta * (i + 1)
func(startday, endday)
return
##################################################
# 时间操作结束
##################################################

##################################################
# 封神榜开始
##################################################

#查询所有消费信息，插入到redis的ACTIVITY:ACTIVITYRANK:USERRANK:6
def alldayUSERRANK(startday):
global gdb_cursor
starttimestamp = time.mktime(startday.timetuple())
query = ("SELECT PRESENTERID, SUM(CONSUMECOIN) AS SUMCOIN FROM ITEMCONSUMERECORD WHERE OPTIME >= 1437148800 and OPTIME < 1439913600 GROUP BY PRESENTERID")
sql = query # % (starttimestamp)
print sql
#try:
gdb_cursor.execute(sql)
dayKey="ACTIVITY:ACTIVITYRANK:USERRANK:6"
deleteKey(dayKey)
for (PRESENTERID, SUMCOIN) in gdb_cursor:
print dayKey, PRESENTERID, SUMCOIN
addzSet(dayKey, PRESENTERID, SUMCOIN)
#finally:
return

#查询消费信息，插入到redis
def everydayUSERRANK(startday, endday):
global gdb_cursor
starttimestamp = time.mktime(startday.timetuple())
endtimestamp = time.mktime(endday.timetuple())
days = startday.strftime('%Y%m%d')
query = ("SELECT PRESENTERID, SUM(CONSUMECOIN) AS SUMCOIN FROM ITEMCONSUMERECORD WHERE OPTIME >= %d AND OPTIME < %d GROUP BY PRESENTERID")
sql = query % (starttimestamp, endtimestamp)
print sql
#try:
gdb_cursor.execute(sql)
dayKey="ACTIVITY:ACTIVITYRANK:USERRANK:6:%s" %days
deleteKey(dayKey)
for (PRESENTERID, SUMCOIN) in gdb_cursor:
print dayKey, PRESENTERID, SUMCOIN
setHash(dayKey, PRESENTERID, SUMCOIN)
#finally:
return

##################################################
# 统计24小时的用户的票数
##################################################
#写日志
def writeLog(txtname, contests):
f = file(txtname,'a')
f.write(contests)
f.close()
return

#获取用户频道昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)
return ret

#获取文件内容
def getLog4Singer(logday):
cmd_ssh = "ssh 172.17.1.125 \"grep -rn \\\"送物品(45)\\\" /home/haoren/logdir/%s_21/activityserver.log* \
\" | awk -F '[()]' '{print $8, $12}'" %logday

# cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(45)\\\" /log/%s/activityserver.log.* \
# |grep \\\"给(61297679)\\\" \
# \" | awk -F '[()]' '{print $8, $12}'" %logday

output = os.popen(str(cmd_ssh))
ret_ssh = output.read()
ret = ret_ssh.strip('\n')
ret = ret.strip(' ')
return ret

def countSigerTick(logname, logday):
#取票
ret_info = getLog4Singer(logday)

#分析
ret_split = ret_info.split()
len_split = len(ret_split)

dic = {}
for i in range (0, len_split/2):
sid = ret_split[ i * 2]
tick = ret_split[i*2 + 1]

dic_tick = dic.get(sid)
if dic_tick is None:
dic[sid] = int(tick)
else:
dic[sid] = int(tick) + int(dic_tick)

return dic
#for sid, tck in dic.items():
# print sid, tck

sort = sorted(dic.items(),key=lambda e:e[1],reverse=True)

#打印
sumTick = 0
contest = ""
for item in sort:
sumTick += int(item[1])
name = getNickname(item[0])
if len(name) == 0:
contest += "SID:%u, ticks:%u, channelNickNam: \r\n" %( int(item[0]), int(item[1]))
else:
contest += "SID:%u, ticks:%u, channelNickNam:%s \r\n" %( int(item[0]), int(item[1]), name[0][0])

contest += "总计赠送了%u票" %sumTick

writeLog(logname, contest)
return

#获取文件内容
def getLog4Singer1(logday):
cmd_ssh = "ssh 172.17.1.125 \"grep -rn \\\"爆星\\\" /home/haoren/logdir/%s_21/activityserver.log.* \" \
|awk -F '[()]' '{print $2, $4}'" %logday

# cmd_ssh = "ssh 172.17.1.21 \"grep -rn \\\"送物品(45)\\\" /log/%s/activityserver.log.* \
# |grep \\\"53014800\\\" \
# \" | awk -F '[()]' '{print $4, $12}'" %logday

output = os.popen(str(cmd_ssh))
ret_ssh = output.read()
ret = ret_ssh.strip('\n')
ret = ret.strip(' ')
#print ret
return ret

def countSigerTick1(logname, logday):
#取票
ret_info = getLog4Singer1(logday)

#分析
ret_split = ret_info.split()
len_split = len(ret_split)

dic = {}
for i in range (0, len_split/2):
sid = ret_split[ i * 2]
tick = ret_split[i*2 + 1]

dic_tick = dic.get(sid)
if dic_tick is None:
dic[sid] = int(tick)
else:
dic[sid] = int(tick) + int(dic_tick)

#for sid, tck in dic.items():
# print sid, tck
return dic

sort = sorted(dic.items(),key=lambda e:e[1],reverse=True)

#打印
sumTick = 0
sumMem = 0
contest = ""
for item in sort:
sumTick += item[1]
sumMem +=1
name = getNickname(item[0])
if len(name) == 0:
contest += "SID:%u, ticks:%u, channelNickNam: \r\n" %( int(item[0]), int(item[1]))
else:
contest += "UID:%u, ticks:%u, channelNickNam:%s \r\n" %( int(item[0]), int(item[1]), name[0][0])

contest += "送物品爆星得到的人数:%u 赠送总票数%u" %(sumMem, sumTick)

writeLog(logname, contest)
return
##################################################
# 统计24小时的用户的票数
##################################################
def main():
mysql_connect()
#获取当天0点的时间戳
today = datetime.date.today()
endTimeStamp = time.mktime(today.timetuple())
startTimeStamp = endTimeStamp - 86400

yesterday = (datetime.datetime.now() - datetime.timedelta(days = 1))
strday = yesterday.strftime("%m%d")

#用户票
logname ="/home/haoren/timingPlan/ceshi/I_want_U/log_sum_tick/"+ strday + "_singerTick.txt"
free_tick = countSigerTick(logname, "15%s" %strday)
luck_tick = countSigerTick1(logname, "15%s"%strday)
for sid, tick in free_tick.items():
luckTick = luck_tick.get(sid)
if luckTick is None:
print "user(%u) is none " % int(sid)
name = getNickname(sid)
contest = "%u,%u,%u,0,%s\r\n" %( int(sid), int(tick), int(tick), name[0][0])
else:
name = getNickname(sid)
   contest = "%u,%u,%u,%u,%s\r\n" %( int(sid), int(tick) + int(luckTick), int(tick), int(luckTick), name[0][0])

   writeLog(logname, contest)
mysql_close()
return
main()

======================================================================================================================
#!/usr/bin/env python
# -*- coding: cp936 -*-

from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import smtplib
import sys
import string

#命令eg: send_mail_attachment.py "test@haoren.com" "test.txt" "/home/haoren/test.txt"
#命令eg: send_mail_attachment.py "test@haoren.com,test1@haoren.com" "test.txt" "/home/haoren/test.txt"
#===================================================================================================
# 注意:发送邮件附件名称,后缀名最好去掉,防止被接收方规则过滤掉
#===================================================================================================

#创建一个带附件的实例

msg = MIMEMultipart()

#获得命令行参数

if 4 != len(sys.argv):
print '参数错误，参考命令:send_mail_attachment.py "test@haoren.com" "test.txt" "/home/haoren/test.txt"'
exit(0)

str_argv = sys.argv[1]
name_attachment = sys.argv[2]
path_attachment = sys.argv[3]

#解析字符串分割成列表

to_lst = string.splitfields(str_argv, ",")

#构造附件1
att1 = MIMEText(open( path_attachment, 'rb').read(), 'base64', 'gb2312')
att1["Content-Type"] = 'application/octet-stream'
att1["Content-Disposition"] = 'attachment; filename="%s"' %name_attachment #这里的filename可以任意写，写什么名字，邮件中显示什么名字
msg.attach(att1)

#构造附件2
#att2 = MIMEText(open('d:\\123.txt', 'rb').read(), 'base64', 'gb2312')
#att2["Content-Type"] = 'application/octet-stream'
#att2["Content-Disposition"] = 'attachment; filename="123.txt"'
#msg.attach(att2)

#############################################################
# #
# 注意：尽量用smtp.163.com 来发送邮件, 否则不一定会成功 #
# #
#############################################################

#邮件配置
mail_host = 'smtp.163.com'
mail_account = '*****@163.com'
mail_passwnd = 'haoren@123.com_z'
mail_postfix = '163.com'

#加邮件头
msg['to'] = str_argv #收件箱内中: 收件人的内容,和实际收到人的无关，只做显示用
msg['from'] = "*******@163.com"
msg['subject'] = '统一邮件' #标题<这里不允许修改，否则会被扔到垃圾箱中>

#发送邮件
try:
server = smtplib.SMTP()
server.connect( mail_host )
server.login( mail_account, mail_passwnd )#XXX为用户名，XXXXX为密码
#server.sendmail(msg['from'], msg['to'], msg.as_string())
server.sendmail(msg['from'], to_lst, msg.as_string())
server.quit()
print '发送成功'
except Exception, e:
print str(e)

======================================================================================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import smtplib
import sys
import string

#命令eg: send_mail_html.py "test@haoren.com" "title1,title2" "/home/haoren/test.txt"
#命令eg: send_mail_html.py "test@haoren.com,test1@haoren.com" "title1,title2" "/home/haoren/test.txt"
#===================================================================================================
# 注意:发送邮件附件名称,后缀名最好去掉,防止被接收方规则过滤掉
#===================================================================================================

#创建一个带附件的实例
msg = MIMEMultipart()

#获得命令行参数
if 4 != len(sys.argv):
print '参数错误，参考命令:send_mail_html.py "test@haoren.com" "title1,title2" "/home/haoren/test.txt"'
exit(0)

str_argv = sys.argv[1]
name_attachment = sys.argv[2]
path_attachment = sys.argv[3]

#解析字符串分割成列表
title_lst = string.splitfields(name_attachment, ",")

#####拼接html#####
html_str = '<html><head><meta charset="UTF-8"></head><body>'
html_str += '<table border=1 borderColor=#D3D8E0 width=1000 align=left >'

#增加标题
if len(title_lst) == 0:
print "标题为0"
exit(0)

html_str += '<tr>'
for lst in title_lst:
html_str += '<td align=center bgColor=#00B2EE>' + json.title[col] + '</td>'
html_str += '</tr>'

#拼接body
file = open( path_attachment, 'rb').read()
for line in file.xreadlines():
print line

html_str += '</table></body></html>'

exit(0)

#构造附件1
att1 = MIMEText(open( path_attachment, 'rb').read(), 'html', 'gb2312')
#att1["Content-Type"] = 'application/octet-stream'
#att1["Content-Disposition"] = 'html; filename="%s"' %name_attachment #这里的filename可以任意写，写什么名字，邮件中显示什么名字
msg.attach(att1)

#构造附件2
#att2 = MIMEText(open('d:\\123.txt', 'rb').read(), 'base64', 'gb2312')
#att2["Content-Type"] = 'application/octet-stream'
#att2["Content-Disposition"] = 'attachment; filename="123.txt"'
#msg.attach(att2)

#############################################################
# #
# 注意：尽量用smtp.163.com 来发送邮件, 否则不一定会成功 #
# #
#############################################################
#邮件配置
mail_host = 'smtp.163.com'
mail_account = '******@163.com'
mail_passwnd = 'haoren@123.com_z'
mail_postfix = '163.com'

#加邮件头
msg['to'] = str_argv #收件箱内中: 收件人的内容,和实际收到人的无关，只做显示用
msg['from'] = "********@163.com"
msg['subject'] = '盾统一邮件' #标题<这里不允许修改，否则会被扔到垃圾箱中>

#发送邮件
try:
server = smtplib.SMTP()
server.connect( mail_host )
server.login( mail_account, mail_passwnd )#XXX为用户名，XXXXX为密码
#server.sendmail(msg['from'], msg['to'], msg.as_string())
server.sendmail(msg['from'], to_lst, msg.as_string())
server.quit()
print '发送成功'
except Exception, e:

print str(e)

=========================================================

#!/usr/bin/python
#-*-coding:utf-8-*-

import sys
import MySQLdb
import datetime
import random
import fnmatch
import random
import os, sys, re,string
import time, tarfile,getopt
import calendar
import fnmatch
import logging

reload(sys)
sys.setdefaultencoding('utf-8')

optmap = {
'dbuser' : 'haoren',
'dbpass' : 'ayFqDxhG4d',
'dbhost' : '172.18.102.8',
'dbport' : 3306,
'dbname' : 'PLIMDB'
}

#日志打印
def initlog():
timestamp = time.time()
timeArray = time.localtime(timestamp)
begin_date = time.strftime("%Y_%m_%d", timeArray)
LOG_PATH = os.getcwd()+'/log/myapp_'+begin_date+'.log'
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename=LOG_PATH,
filemode='w')

initlog()

# logging.basicConfig(level=logging.INFO,
# filename='./AutoSettle.log',
# filemode='w',
# format='%(asctime)s - %(filename)s:%(lineno)s[%(funcName)s] - %(levelname)s: %(message)s')
#

#数据库连接，执行
def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()
db_cursor.close()
db_conn.close
logging.info("连接数据库")
return ret

except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
logging.error("错误")
return None

#遍历文件夹
def get_files(dir, pattern):
res_file_list =[]
if os.path.exists(dir):
cur_file_list = os.listdir(dir)
for file_name in cur_file_list:
if re.search(pattern, file_name):
res_file_list.append(file_name)
return res_file_list
else:
return 'no'

#查昵称
def getNickname(sid):
id = int(sid)%10
sql = "select CHANNELNICKNAME from CHARBASE%u where `ID`=%u" %(id, int(sid))
ret = sql_select(sql)
return ret

#保存到文件
def writeLog(txtname, contests):
strTmp = time.strftime('%Y%m%d')
ret = os.path.exists('/home/haoren/y/%s' %strTmp)
if ret == False:
os.mkdir('/home/haoren/y//%s' %strTmp)
f = file("/home/haoren/y/%s/%s" %(strTmp,txtname),'a')
f.write(contests)
f.close()
return

begin = datetime.date(2017,8,10)
end = datetime.date(2017,8,10)
#print "difference :",(end - begin).days
userlist = {}

strSinger = "Singer.txt"

for i in range((end - begin).days+1):

day = begin + datetime.timedelta(days=i)
print "second :" ,str(day)
day1 = day.strftime('%y%m%d')
print "day1=" ,day1
dirname = "/home/haoren/logdir/%s_34/" %day1
print dirname
  ret = get_files(dirname,'gameappserver')
ret.sort()
#print ret

for file2 in ret:
print file2

f = open(dirname+file2 , 'r')
#line = f.readline()
#line = line[:-1]
#print line
#userlist = []
  for a_line in f.readlines():
#170703-00:00:26 GameAppServer[17400] INFO: [GiftRoll2] [10461539]用户(21441172)增加人民币(1000)成功，剩余人民币(16377)
m = re.search("^(\S+) GameAppServer\[\d+\] INFO: \[GiftRoll2\] \[\d+\]用户$(\d+)$增加人民币$(\d+)$成功，剩余人民币$(\d+)$",a_line)
if m:
#print m.group(0)
singerid = int(m.group(2))
gold = int(m.group(3))
if (singerid in userlist):
userlist[singerid] += gold
else:
userlist[singerid] = gold


#print "第一次打印字典\n"
#print userlist.items()

#print "第二次打印字典\n"
#for k in userlist :
# print "userlist[%s] =" %k,userlist[k]

#print "第三次打印字典\n"
#for (k ,v) in userlist.items():
# print "userlist[%s] =" %k,v

#print "第四次打印字典"
#print userlist.iteritems()
#for (k,v) in userlist.iteritems():
# print "userlist[%s]=" %k,v

#print "第五次打印字典\n"
#for (k,v) in zip(userlist.iterkeys(),userlist.itervalues()):
#
# print "userlist[%s]=" %k,v

#print "第六次打印字典\n"
#for (k ,v) in userlist.items():
# print k,v

#按key排序
#print sorted(userlist.keys())

#按value排序
t = sorted(userlist.items(),key=lambda item:item[1],reverse=True)

n=1
for m in t:
nickname = getNickname(int(m[0]))
#print n, m[0], ",", m[1],",",nickname[0][0].decode('gbk')
#print n, m[0], ",", m[1],",",nickname[0][0]
#contest="序列号:%s:UID:%s:金额:%s:昵称:%s \r\n" %(n, m[0], m[1], nickname[0][0])
contest="序列号:%s \t:UID:%s \t:金额:%s \t:昵称:%s \r\n" %(n, m[0], m[1], nickname[0][0])
writeLog(strSinger,contest)
n += 1
if n > 100 :
break

=================================================================================

%pyspark
#查询认证用户
import sys
#import MySQLdb
import mysql.connector
import pandas as pd
import datetime
import time

optmap = {
'dbuser' : 'haoren',
'dbpass' : 'G4d',
'dbhost' : '172.12.112.8',
'dbport' : 3306,
'dbname' : 'IMDB'
}

def sql_select(reqsql):
ret = ''
try:
db_conn = mysql.connector.connect(user=optmap['dbuser'], password=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], database=optmap['dbname'])
db_cursor=db_conn.cursor()
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()
except mysql.connector.Error as e:
print ('Error : {}'.format(e))
finally:
db_cursor.close()
db_conn.close
return ret

def renzhengsingger(startday,endday):   #定义查询认证用户函数
t1 = int(time.mktime(time.strptime(startday,'%Y-%m-%d %H:%M:%S')) )
t2 = int(time.mktime(time.strptime(endday,'%Y-%m-%d %H:%M:%S')))
reqsql = "select PERFORMERID,from_unixtime(ADDTIME) from PERFORMERINFO where ADDTIME >=%s and ADDTIME < %s" %(t1,t2)
ret = sql_select(reqsql)
for i in ret:
print i[0],",",i[1]


renzhengsingger('2017-08-01 00:00:00','2017-09-01 00:00:00') #调用函数