traffic and backfill

#!/usr/bin/python
#coding=utf-8
import time
import re
import os
#############################################
# Name: TRAFFIC AND BACKFILL    
# Author: LCY
# Version: 1.0.20150828
# Feature: xxxxxx
#############################################




#appoint sgwip and report the traffic
def check_traffic():
    TODAY14 = get_endtime()
    YESTERDAY14 = get_starttime()
    os.system("impala-shell -q \"refresh pqtxdr.lteu1_ftp_zc ; refresh pqtxdr.lteu1_rtsp_zc ; refresh pqtxdr.lteu1_s1u_zc ; refresh pqtxdr.lteu1_voip_zc ; refresh pqtxdr.lteu1_http_zc ;  with h2 as (select     startdate,u32sgwip_4,u32ultraffic,u32dltraffic from pqtxdr.lteu1_ftp_zc where startdate>=\'"+YESTERDAY14+ "\' and startdate<=\'"+TODAY14+"\'), h3 as (select startdate,u32sgwip_4,u32ultraffic,u32dltraffic from     pqtxdr.lteu1_rtsp_zc where startdate>=\'"+YESTERDAY14+ "\' and startdate<=\'"+TODAY14+"\'), h4 as (select startdate,u32sgwip_4,u32ultraffic,u32dltraffic from pqtxdr.lteu1_s1u_zc where startdate>=\'"+YESTERDAY14+     "\' and startdate<=\'"+TODAY14+"\'), h5 as (select startdate,u32sgwip_4,u32ultraffic,u32dltraffic from pqtxdr.lteu1_voip_zc where startdate>=\'"+YESTERDAY14+ "\' and startdate<=\'"+TODAY14+"\'), h6 as (select     startdate,u32sgwip_4,u32ultraffic,u32dltraffic from pqtxdr.lteu1_http_zc where startdate>=\'"+YESTERDAY14+ "\' and startdate<=\'"+TODAY14+"\'), h21 as ( select * from h2 union all select * from h3 union all select *     h4 union all select * from h5 union all select * from h6 ), h22 as (select startdate,u32sgwip_4,SUM(u32ultraffic) as ul_traffic,SUM(u32dltraffic) as dl_traffic,sum(u32ultraffic+u32dltraffic) as total_traffic from h21 group by     startdate,u32sgwip_4) select h22.startdate,h22.u32sgwip_4,h22.ul_traffic,h22.dl_traffic,h22.total_traffic from h22 ;\" > /tmp/Log_traffic.log")
    sgwip=open('/tmp/sgwip','r')#path of sgwip
    lines=sgwip.readlines()
    for l in lines:
        list=l.split()
        IP=list[0]
        NAME=list[1]
        print IP,NAME
        os.chdir('/tmp/')##save path
        name=open(NAME+'tmp','w+')
        log=open('/tmp/Log_traffic.log','r')#import the ip,traffic in this file
        lines_log=log.readlines()
        for L in lines_log:
        match=re.search(IP,L)
        if match:
                plit_line=L.split('|')
                del split_line[3]
                del split_line[3]
                str=''.join(split_line)
                name.write(str)
        name.close()


    finalfile=open(NAME,'w')
    finalname=open(NAME+'tmp','r')
    lines_finalfile=finalname.readlines()
    lines_finalfile.sort()
    for l_finalfile in lines_finalfile:
        finalfile.write(l_finalfile)
        os.remove(NAME+'tmp')
        print '%s download success'%NAME





#get start and  end time
def get_endtime():
        TODAY=time.strftime('%Y%m%d',time.localtime(time.time()))
        CHECKTIME=TODAY+"14"
        return CHECKTIME


def get_starttime():
        YESTERDAY=time.strftime('%Y%m%d',time.localtime(time.time()-24*3600))
        #time.time()
        CHECKTIME=YESTERDAY+"14"
        return CHECKTIME


def check_backfill():
        try:
            CITYCORE=sys.argv[1]
        except Exception,e:
            CITYCORE=25
        print "CITYCORE=",CITYCORE
        TODAY14=get_endtime()
        YESTERDAY14=get_starttime()
        os.system("impala-shell -q"+"refresh pqtxdr.lteu1_http_zc;select startdate,'imsi' as type,count(*) from pqtxdr.lteu1_http_zc where startdate>=" +YESTERDAY14+ " and startdate<="+TODAY14+ " and         u64imsi>1000000000000 and u16citycode=" +CITYCORE+ " group by startdateunion all select startdate,'msisdn' as type,count(*) from lteu1_http_zc where startdate>="+YESTERDAY14+ " and startdate<="+TODAY14+         "and u64msisdn>13000000000 and u16citycode="+CITYCORE+" group by startdateunion allselect startdate,'count_cnt' as type,count(*) from lteu1_http_zc where startdate>="+YESTERDAY14+ " and         startdate<="+TODAY14+ " and u16citycode=25 group by startdate"+" >/tmp/Log_backfill.log")
        os.chdir('/tmp/')
        tmpfile=open('Log_backfill.log','r')
        tmpfile_lines=tmpfile.readlines()
        tmpfile2=open('imsitmp','w')
        tmpfile3=open('msisdntmp','w')
        tmpfile4=open('count_cnttmp','w')
        for tmpfile_line in tmpfile_lines:
                match1=re.search(r'imsi',tmpfile_line)
                if match1:
           tmpfile2.write(tmpfile_line)
                match2=re.search(r'msisdn',tmpfile_line)
                if match2:
                        tmpfile3.write(tmpfile_line)
                match3=re.search(r'count_cnt',tmpfile_line)
                if match3:
                        tmpfile4.write(tmpfile_line)
        tmpfile2.close()
        tmpfile3.close()
        tmpfile4.close()
        tmpfile5=open('imsitmp','r')
        tmpfile6=open('msisdntmp','r')
        tmpfile7=open('count_cnttmp','r')
        tmpfile8=open('imsi','w')
        tmpfile9=open('msisdn','w')
        tmpfile0=open('count_cnt','w')
        tmpfile5_line=tmpfile5.readlines()
        tmpfile5_line.sort()
        for tmpfile5_line_l in tmpfile5_line:
                tmpfile8.write(tmpfile5_line_l)
        os.remove('imsitmp')
        print 'imsi success'
        tmpfile6_line=tmpfile6.readlines()
        tmpfile6_line.sort()
        for tmpfile6_line_l in tmpfile6_line:
                tmpfile9.write(tmpfile6_line_l)
        os.remove('msisdntmp')
        print 'msisdn success'
        tmpfile7_line=tmpfile7.readlines()
        tmpfile7_line.sort()
        for tmpfile7_line_l in tmpfile7_line:
               tmpfile0.write(tmpfile7_line_l)
        os.remove('count_cnttmp')
        print 'count_cnt success'


if __name__ == '__main__':
    get_endtime()
    get_starttime()
    check_traffic()
    check_backfill()