import subprocess, shlex, os, time
from StringIO import StringIO
import whatap.util.file_util as file_util
from whatap.util.ip_util import IPUtil as ip_util
from whatap.io.data_outputx import DataOutputX
import whatap.util.process_util as process_util
import whatap.util.logging_util as logging_util
import whatap.util.thread_util as thread_util
import whatap.agent.conf.configure as conf
import psutil
import threading
import multiprocessing
import copy
import procgroup_rule as procgroup_rule

cpucore = float(multiprocessing.cpu_count())


class ProcessPerfInfo:
    def __init__(self):
        self.ppid = None
        self.pid = None
        self.cpu = None
        self.cpuUser = 0.0
        self.cpuSys = 0.0
        self.memoryBytes = None
        self.memoryPercent = None
        self.vsz = 0
        self.readBps = 0.0
        self.writeBps = 0.0
        self.cmd1 = None
        self.cmd2 = None
        self.readIops = 0.0
        self.writeIops = 0.0
        self.user = None
        self.state = None
        self.name = None
        self.createTime = None
        self.net = []
        self.file = []
        self.sharedMemory = 0
        self.threadCount = 0
        self.processCount = 0
        self.totalCharIO = 0
        self.pids = []

    def __str__(self):
        b = StringIO()
        b.write('ppid:')
        b.write(self.ppid)
        b.write('\npid')
        b.write(self.pid)
        b.write('\ncpu')
        b.write(self.cpu)
        b.write('\nmemory')
        b.write(self.memoryBytes)
        b.write('\nsharedMemory')
        b.write(self.sharedMemory)


        return b.getvalue()


class IpPort:
    def __init__(self, localip=0, localport=0, conn=0, ilocaladdr=0, inode=0):
        self.localip = localip
        self.localport = localport
        self.ilocaladdr = ilocaladdr
        self.iremoteaddr = 0
        self.conn = conn
        self.inode = inode

    def __str__(self):
        buf = StringIO()
        buf.write("localip=")
        buf.write(self.localip)
        buf.write("\nlocalport=")
        buf.write(self.localport)
        buf.write("\nilocaladdr=")
        buf.write(self.ilocaladdr)
        buf.write("\niremoteaddr=")
        buf.write(self.iremoteaddr)
        buf.write("\nconn=")
        buf.write(self.conn)
        buf.write("\ninode=")
        buf.write(self.inode)

        return buf.getvalue()


class FileInfo:
    def __init__(self, name=None, size=0):
        self.name = name
        self.size = size

def populateNetworkStatus(processPerfs):
    localips = []
    ipinfo = process_util.executeCmd("ip addr show")

    for line in ipinfo.split('\n'):
        words = line.split()

        if words and words[0] == 'inet':
            localips.append(words[1].split('/')[0])
    inodeportlookup = {}
    ipinodelookup = {}
    remoteconns = []
    for tcpfilepath in ("/proc/net/tcp", "/proc/net/tcp6"):
        tcpContent = file_util.readContent(tcpfilepath)
        if tcpContent:
            i = 0
            for line in tcpContent.split("\n"):
                i += 1
                if i == 1 or not line:
                    continue

                words = line.split()
                inode = int(words[9])
                localIPBytes = ip_util.parseHexString(words[1])
                if len(localIPBytes) != 6:
                    continue
                localIP = DataOutputX.toInt(localIPBytes[:4])
                localPort = DataOutputX.toShort(localIPBytes[4:])
                localIPPortInt = DataOutputX.toLong6(localIPBytes)
                remoteIPBytes = ip_util.parseHexString(words[2])
                if len(remoteIPBytes) != 6:
                    continue
                remoteIPInt = DataOutputX.toInt(remoteIPBytes[:4])
                remotePortInt = DataOutputX.toShort(remoteIPBytes[4:])
                ipport = IpPort(localip=localIP, localport=localPort, conn=0, ilocaladdr=localIPPortInt,
                                inode=int(inode))
                if remoteIPInt == 0 and remotePortInt == 0:
                    inodeportlookup[inode] = ipport
                    if localIPPortInt > 0:
                        ipinodelookup[localIPPortInt] = ipport
                        for ip in localips:
                            ipbytes = ip_util.toBytes(ip)
                            deviceIPPortBytes = StringIO()
                            deviceIPPortBytes.write(ipbytes)
                            deviceIPPortBytes.write(localIPBytes[4:])
                            deviceIPPortInt = DataOutputX.toLong6(deviceIPPortBytes.getvalue())
                            ipinodelookup[deviceIPPortInt] = ipport

                else:
                    remoteconns.append(ipport)

    for pipport in remoteconns:
        if pipport.ilocaladdr in ipinodelookup:
            ipinodelookup[pipport.ilocaladdr].conn += 1

    # print 'ipinodelookup'
    # from pprint import pprint
    # for k, v in ipinodelookup.items():
    #    print k,v

    for processPerf in processPerfs:
        ports = []
        filesinfo = []
        searchDir = os.path.join("/proc", str(processPerf.pid), "fd")

        if not os.path.exists(searchDir):
            continue
        for filename in os.listdir(searchDir):
            path = os.path.join(searchDir, filename)

            if os.path.islink(path):
                link = os.readlink(path)

                if link.startswith("socket:["):
                    inode = int(link[8:len(link) - 1])
                    # print 'inode', inode, inode in inodeportlookup
                    if inode in inodeportlookup:
                        ipport = inodeportlookup[inode]
                        ports.append(IpPort(localip=ipport.localip, localport=ipport.localport, conn=ipport.conn))

                elif len(link) > 0 and not link.startswith("/dev") and link[0] == '/':
                    try:
                        filestat = os.stat(link)
                        # print 'file ',processPerf.cmd1, link, filestat.st_size
                        if filestat and link.endswith(".log"):
                            fileinfo = FileInfo(name=link, size=filestat.st_size)
                            filesinfo.append(fileinfo)
                            # print processPerf.cmd1, link
                    except Exception, e:
                        pass
        processPerf.net = ports
        processPerf.file = filesinfo

class _ProcPerfCache:
    def __init__(self, cpu_times=None, proc_map=None, timestamp=None):
        self.cpu_times = cpu_times
        self.proc_map = proc_map
        if timestamp:
            self.timestamp = timestamp
        else:
            self.timestamp = time.time()


jiff = 100

def parseNetwork(conns):
    port_map = {}
    ports= []
    for c in conns:
        if c.family == 2 and c.type == 2 and c.status=='LISTEN':
            ipport = IpPort(localip=ip_util.strToInt(c.laddr.ip), localport=c.laddr.port, conn=0)
            port_map[c.laddr.port] = ipport
            ports.append(ipport)
    for c in conns:
        if c.family == 2 and c.type == 2 and c.status == 'ESTABLISHED' and c.laddr.port in port_map:
            port_map[c.laddr.port].conn += 1
    return ports

def parseOpenFile(files):
    flookup = {}
    fileusage =[]
    for f in files:
        if f.path and f.path.endswith('.log'):
            flookup[f.path] = 0
    for p in flookup.keys():
        filestat = os.stat(p)
        if filestat:
            fileusage.append(FileInfo(name=p, size=filestat.st_size))

    return fileusage



proc_shm_cache = {}
data_map_lock = threading.Lock()
update_condition = threading.Condition()
update_queue = []

def updateShmCache():
    while True:
        with update_condition:
            while not update_queue:
                update_condition.wait()

            proc = update_queue.pop(0)

        with data_map_lock:
            sharedMemory = 0
            try:
                if proc.pid not in proc_shm_cache or proc.create_time() != proc_shm_cache[proc.pid][1]:
                    for m in proc.memory_maps():
                        if m.path == '[shmid]':
                            sharedMemory += m.rss
                proc_shm_cache[proc.pid] = (sharedMemory, proc.create_time())
            except psutil.ZombieProcess, e:
                logging_util.debug("zombie process skip", e)
                continue
            except psutil.NoSuchProcess, e: 
                logging_util.debug("process cpu time error : ", e)
                continue


def setSharedMemory(proc):
    if data_map_lock.acquire(False):
        try:
            if proc.pid in proc_shm_cache and proc.create_time() == proc_shm_cache[proc.pid][1]:
                (sharedMemory, _) = proc_shm_cache[proc.pid]
                return sharedMemory
            else:
                with update_condition:
                    update_queue.append(proc)
                    update_condition.notify()
                return 0

        finally:
            data_map_lock.release()
    else:
        with update_condition:
            update_queue.append(proc)
            update_condition.notify()
        return 0


oldProcPerfCache = None



PROC_STATUSES = {
    psutil._pssunos.cext.SSLEEP: psutil._common.STATUS_SLEEPING,
    psutil._pssunos.cext.SRUN: psutil._common.STATUS_RUNNING,
    psutil._pssunos.cext.SZOMB: psutil._common.STATUS_ZOMBIE,
    psutil._pssunos.cext.SSTOP: psutil._common.STATUS_STOPPED,
    psutil._pssunos.cext.SIDL: psutil._common.STATUS_IDLE,
    psutil._pssunos.cext.SONPROC: psutil._common.STATUS_RUNNING,  # same as run
    psutil._pssunos.cext.SWAIT: psutil._common.STATUS_WAITING,
}


def getProcessPerfList():
    global proc_shm_cache
    proc_map = {}

    vmem = psutil.virtual_memory()
    totalMem = vmem.total
    proc_list = []

    cpu_times = psutil.cpu_times(percpu=False)
    cpu_jiff_diff = 0

    global oldProcPerfCache
    if oldProcPerfCache:
        all_cpu = time.time()
        last_all_cpu = oldProcPerfCache.timestamp
        cpu_jiff_diff = float(all_cpu - last_all_cpu) * cpucore

    try:
        procInfo = psutil.proc_detail_info()
        #0 pid
        #1 ppid
        #2 name
        #3 cmdline
        #4 exe
        #5 username
        #6 rss
        #7 vms
        #8 createtime
        #9 thread count
        #10 status code
        #11 utime
        #12 stime
        #13 ioch
        #14 inblk
        #15 oublk
        for proc in procInfo.values():
            now_proc_times = proc[11] + proc[12]
            pid = proc[0]
            proc_map[pid] = now_proc_times

            if not oldProcPerfCache:
                continue

            if pid not in oldProcPerfCache.proc_map:
                continue

            p = ProcessPerfInfo()
            p.pid = pid
            p.ppid = proc[1]
            p.user = proc[5]
            
            old_proc_times = oldProcPerfCache.proc_map[pid]
            if cpu_jiff_diff != 0:
                p.cpu = float(100) * float(now_proc_times - old_proc_times) / cpu_jiff_diff

            p.memoryBytes = proc[6]
            p.memoryPercent = 100.0 * p.memoryBytes / totalMem
            p.cmd1 = proc[4]
            p.cmd2 = proc[3]
            p.createTime = int(proc[8])

            p.state = PROC_STATUSES.get(proc[10], '?')

            #TODO 
            if conf.GetConfig().serverProcessConnectCheck:
                p.net = parseNetwork(psutil._pssunos.net_connections('inet', pid))
            #TODO 
            if conf.GetConfig().serverProcessFDCheck:
                processObj = psutil.Process(pid)
                p.file = parseOpenFile(processObj.open_files())

            proc_list.append(p)
    except Exception as e:
        logging_util.error("proc(perfList) detail error : ", e)
        return None

    procPerf = _ProcPerfCache(cpu_times = cpu_times, proc_map = proc_map)
    oldProcPerfCache = procPerf
    return proc_list

init=False

def GetProc():
    global init
    if init:
        thread_util.async(updateShmCache)
        init=False

    logging_util.debug("Get Proc Start")
    procPerfs = getProcessPerfList()
    if procPerfs == None:
        logging_util.debug("Get Proc End : None")
    else:
        logging_util.debug("Get Proc End : ", len(procPerfs))

    return procPerfs




def GetTotalProc():
    try:
        proc, thread, defunct = psutil.proc_total_info()
        return (proc, thread, defunct)
    except:
        return (None, None, None)


oldGroupProcPerfCache = None

def GetTopNAndGroupProc(n):
    cpu_threshold = 0.1
    rss_threshold = 20 * 1024 * 1024

    cpu_topn = n / 4 * 3
    memory_topn = n / 4 

    proc_map = {}
    proc_list = []

    cpu_times = psutil.cpu_times(percpu=False)
    cpu_jiff_diff = 0
    timediff = 0
    now = time.time()
    global oldGroupProcPerfCache 
    if oldGroupProcPerfCache:
        oldTime = oldGroupProcPerfCache.timestamp
        timediff = float(now - oldTime)
        cpu_jiff_diff = timediff * cpucore


    try:
        procInfo = psutil.proc_detail_info()

        #0 pid
        #1 ppid
        #2 name
        #3 cmdline
        #4 exe
        #5 username
        #6 rss
        #7 vms
        #8 createtime
        #9 thread count
        #10 status code
        #11 utime
        #12 stime
        #13 ioch
        #14 inblk
        #15 oublk
        for proc in procInfo.values():
            now_utime = proc[11]
            now_stime = proc[12]
            now_proc_times = now_utime + now_stime
            total_char_io = proc[13]
            pid = proc[0]

            proc_map[pid] = (now_utime, now_stime, total_char_io)

            if not oldGroupProcPerfCache:
                continue

            if pid not in oldGroupProcPerfCache.proc_map:
                continue

            p = ProcessPerfInfo()
            p.pid = pid
            p.ppid = proc[1]
            p.user = proc[5]

            old_entry = oldGroupProcPerfCache.proc_map[pid]
            old_utime = old_entry[0]
            old_stime = old_entry[1]
            old_total_char_io = old_entry[2] if len(old_entry) > 2 else old_entry[1]
            old_proc_times = old_utime + old_stime

            if cpu_jiff_diff != 0:
                p.cpu = float(100) * float(now_proc_times - old_proc_times) / cpu_jiff_diff
                p.cpuUser = float(100) * float(now_utime - old_utime) / cpu_jiff_diff
                p.cpuSys = float(100) * float(now_stime - old_stime) / cpu_jiff_diff
            else:
                p.cpu = 0.0
                p.cpuUser = 0.0
                p.cpuSys = 0.0

            p.totalCharIO = float(total_char_io - old_total_char_io) / float(timediff)
            p.memoryBytes = proc[6]
            p.vsz = proc[7]
            p.cmd2 = proc[3]
            p.cmd1 = proc[3].split(' ')[0] if proc[3] else proc[2]
            p.createTime = int(proc[8])
            p.threadCount = proc[9]
            p.state = PROC_STATUSES.get(proc[10], '?')
            p.processCount = 1
            p.pids = [pid]

            proc_list.append(p)
    except Exception as e:
        logging_util.error("proc(TopN_Group) detail error : ", e)
        return None, None

    procPerf = _ProcPerfCache(cpu_times = cpu_times, proc_map = proc_map, timestamp = now)
    oldGroupProcPerfCache = procPerf
    procgroup_rule.gc_pid_cache(set(proc_map.keys()))
    cpu_top = sorted([proc for proc in proc_list if proc.cpu > cpu_threshold], key = lambda p: p.cpu, reverse = True)[:cpu_topn]
    cpu_top_pids = {proc.pid for proc in cpu_top}

    rss_top = sorted([proc for proc in proc_list if proc.pid not in cpu_top_pids and proc.memoryBytes > rss_threshold], key = lambda p: p.memoryBytes, reverse = True)[:memory_topn]

    topn = cpu_top + rss_top

    process_group = {}

    for proc in proc_list:
        if proc.state == 'zombie':
            key = ('defunct', proc.user)
            group_name = 'defunct'
        else:
            matched = procgroup_rule.find_custom_group(proc.pid, proc.user, proc.cmd2)
            if matched:
                key = (proc.user, matched)
                group_name = matched
            else:
                key = (proc.cmd1, proc.user)
                group_name = None

        if key not in process_group:
            p = copy.deepcopy(proc)
            p.name = group_name if group_name else proc.cmd1
            process_group[key] = p
        else:
            process_group[key].cpu += proc.cpu
            process_group[key].cpuUser += proc.cpuUser
            process_group[key].cpuSys += proc.cpuSys
            process_group[key].memoryBytes += proc.memoryBytes
            process_group[key].vsz += proc.vsz
            process_group[key].threadCount += proc.threadCount
            process_group[key].processCount += 1
            process_group[key].totalCharIO += proc.totalCharIO
            process_group[key].pids.extend(proc.pids)

    return (topn, process_group)


def test():
    while True:
        procs = GetProc()
        print
        procs
        if not procs:
            time.sleep(5)
            continue
        time.sleep(5)


if __name__ == '__main__':
    test()
