import subprocess, shlex, os, time
from StringIO import StringIO
import whatap.util.file_util as file_util
from whatap.util.ip_util import IPUtil as ip_util
from whatap.io.data_outputx import DataOutputX
import whatap.util.process_util as process_util
import whatap.util.thread_util as thread_util
import whatap.util.logging_util as logging_util
import whatap.agent.conf.configure as conf
import heapq
import procgroup_rule

import psutil

import system_info_aix

cpucore = float(system_info_aix.cpu_core)


class ProcessPerfInfo:
    def __init__(self):
        self.ppid = None
        self.pid = None
        self.cpu = None
        self.cpuUser = 0.0
        self.cpuSys = 0.0
        self.memoryBytes = None
        self.memoryPercent = None
        self.readBps = 0.0
        self.writeBps = 0.0
        self.cmd1 = None
        self.cmd2 = None
        self.readIops = 0.0
        self.writeIops = 0.0
        self.user = None
        self.state = None
        self.name = None
        self.createTime = None
        self.net = []
        self.file = []
        self.sharedMemory = 0
        self.pids = []

    def __str__(self):
        b = StringIO()
        b.write('ppid:')
        b.write(self.ppid)
        b.write('\npid')
        b.write(self.pid)
        b.write('\ncpu')
        b.write(self.cpu)
        b.write('\nmemory')
        b.write(self.memoryBytes)
        b.write('\nsharedMemory')
        b.write(self.sharedMemory)


        return b.getvalue()


class IpPort:
    def __init__(self, localip=0, localport=0, conn=0, ilocaladdr=0, inode=0):
        self.localip = localip
        self.localport = localport
        self.ilocaladdr = ilocaladdr
        self.iremoteaddr = 0
        self.conn = conn
        self.inode = inode

    def __str__(self):
        buf = StringIO()
        buf.write("localip=")
        buf.write(self.localip)
        buf.write("\nlocalport=")
        buf.write(self.localport)
        buf.write("\nilocaladdr=")
        buf.write(self.ilocaladdr)
        buf.write("\niremoteaddr=")
        buf.write(self.iremoteaddr)
        buf.write("\nconn=")
        buf.write(self.conn)
        buf.write("\ninode=")
        buf.write(self.inode)

        return buf.getvalue()


class FileInfo:
    def __init__(self, name=None, size=0):
        self.name = name
        self.size = size

def populateNetworkStatus(processPerfs):
    localips = []
    ipinfo = process_util.executeCmd("ip addr show")

    for line in ipinfo.split('\n'):
        words = line.split()

        if words and words[0] == 'inet':
            localips.append(words[1].split('/')[0])

    inodeportlookup = {}
    ipinodelookup = {}
    remoteconns = []
    for tcpfilepath in ("/proc/net/tcp", "/proc/net/tcp6"):
        tcpContent = file_util.readContent(tcpfilepath)
        if tcpContent:
            i = 0
            for line in tcpContent.split("\n"):
                i += 1
                if i == 1 or not line:
                    continue

                words = line.split()
                inode = int(words[9])
                localIPBytes = ip_util.parseHexString(words[1])
                if len(localIPBytes) != 6:
                    continue
                localIP = DataOutputX.toInt(localIPBytes[:4])
                localPort = DataOutputX.toShort(localIPBytes[4:])
                localIPPortInt = DataOutputX.toLong6(localIPBytes)
                remoteIPBytes = ip_util.parseHexString(words[2])
                if len(remoteIPBytes) != 6:
                    continue
                remoteIPInt = DataOutputX.toInt(remoteIPBytes[:4])
                remotePortInt = DataOutputX.toShort(remoteIPBytes[4:])
                ipport = IpPort(localip=localIP, localport=localPort, conn=0, ilocaladdr=localIPPortInt,
                                inode=int(inode))
                if remoteIPInt == 0 and remotePortInt == 0:
                    inodeportlookup[inode] = ipport
                    if localIPPortInt > 0:
                        ipinodelookup[localIPPortInt] = ipport
                        for ip in localips:
                            ipbytes = ip_util.toBytes(ip)
                            deviceIPPortBytes = StringIO()
                            deviceIPPortBytes.write(ipbytes)
                            deviceIPPortBytes.write(localIPBytes[4:])
                            deviceIPPortInt = DataOutputX.toLong6(deviceIPPortBytes.getvalue())
                            ipinodelookup[deviceIPPortInt] = ipport

                else:
                    remoteconns.append(ipport)

    for pipport in remoteconns:
        if pipport.ilocaladdr in ipinodelookup:
            ipinodelookup[pipport.ilocaladdr].conn += 1

    # print 'ipinodelookup'
    # from pprint import pprint
    # for k, v in ipinodelookup.items():
    #    print k,v

    for processPerf in processPerfs:
        ports = []
        filesinfo = []
        searchDir = os.path.join("/proc", str(processPerf.pid), "fd")

        if not os.path.exists(searchDir):
            continue
        for filename in os.listdir(searchDir):
            path = os.path.join(searchDir, filename)

            if os.path.islink(path):
                link = os.readlink(path)

                if link.startswith("socket:["):
                    inode = int(link[8:len(link) - 1])
                    # print 'inode', inode, inode in inodeportlookup
                    if inode in inodeportlookup:
                        ipport = inodeportlookup[inode]
                        ports.append(IpPort(localip=ipport.localip, localport=ipport.localport, conn=ipport.conn))

                elif len(link) > 0 and not link.startswith("/dev") and link[0] == '/':
                    try:
                        filestat = os.stat(link)
                        # print 'file ',processPerf.cmd1, link, filestat.st_size
                        if filestat and link.endswith(".log"):
                            fileinfo = FileInfo(name=link, size=filestat.st_size)
                            filesinfo.append(fileinfo)
                            # print processPerf.cmd1, link
                    except Exception, e:
                        logging_util.debugStack(e)
        processPerf.net = ports
        processPerf.file = filesinfo

class _ProcPerfCache:
    def __init__(self, cpu_times=None, proc_map=None, timestamp=None):
        self.cpu_times = cpu_times
        self.proc_map = proc_map
        if timestamp:
            self.timestamp = timestamp
        else:
            self.timestamp = time.time()



oldProcPerfCache = {}
import socket
def parseNetwork(conns):
    port_map = {}
    ports= []
    if conf.GetConfig().serverProcessFDCheck:
        for c in conns:
            if c.family in (socket.AF_INET, socket.AF_INET6) and \
                c.type in (socket.SOCK_STREAM, socket.SOCK_DGRAM) and c.status=='LISTEN':
                ipport = IpPort(localip=ip_util.toInt(c.laddr.ip), localport=c.laddr.port, conn=0)
                port_map[c.laddr.port] = ipport
                ports.append(ipport)
        for c in conns:
            if c.family in (socket.AF_INET, socket.AF_INET6) and \
                c.type in (socket.SOCK_STREAM, socket.SOCK_DGRAM) and c.status=='ESTABLISHED' and \
                c.laddr.port in port_map:
                port_map[c.laddr.port].conn += 1
    return ports

def parseOpenFile(files):
    flookup = {}
    fileusage =[]
    if conf.GetConfig().serverProcessFDCheck:
        for f in files:
            if f.path and f.path.endswith('.log'):
                flookup[f.path] = 0
        for p in flookup.keys():
            filestat = os.stat(p)
            if filestat:
                fileusage.append(FileInfo(name=p, size=filestat.st_size))

    return fileusage

proc_shm_cache = {}

def calcPs(v, lastv, timediff):

    return float(v - lastv) / float(timediff)


processInfoCache = {}
lastCacheClear = None
def getProcessInfo(proc, proc_map):
    now = datetime.now()
    global lastCacheClear
    if lastCacheClear and now - lastCacheClear > timedelta(minutes=90):
        lastCacheClear = now
        processInfoCache.clear()
    pid = proc.pid
    create_time = proc.create_time()

    if pid in processInfoCache:
        username, cmd1, cmd2, target_create_time, ppid, proc_status = processInfoCache[pid]
        if target_create_time == create_time:
            return username, cmd1, cmd2, ppid, proc_status, create_time
    username = proc.username()
    cmd1 = proc.exe()
    cmd2 = ' '.join(proc.cmdline())
    ppid = proc.ppid()
    proc_status = proc.status()
    if not cmd1:
        cmd1 = proc.name()
    processInfoCache[pid] = (username, cmd1, cmd2, create_time, ppid, proc_status)

    return username, cmd1, cmd2, ppid, proc_status, create_time

def getProcessPerfList_deprecate(pidPageSize=10, sleepTime=1):
    global proc_shm_cache
    cpu_times = psutil.cpu_times(percpu=False)
    proc_map = {}
    for i, proc in enumerate(psutil.process_iter()):
        thread_util.cpuOverloadProtect(i, pidPageSize, sleepTime=sleepTime)
        try:
            timestamp = time.time()
            iocounters = None
            if conf.GetConfig().serverProcessFDCheck and hasattr(proc, 'io_counters'):
                try:
                    if conf.process_fdcheck:
                        iocounters =proc.io_counters()
                except OSError:
                    pass
            proc_map[proc.pid] = (proc, proc.cpu_times(), iocounters, timestamp)
        except psutil.NoSuchProcess, e:
            pass
    procPerfCache = _ProcPerfCache(cpu_times=cpu_times, proc_map=proc_map)
    global oldProcPerfCache
    if not oldProcPerfCache:
        oldProcPerfCache = procPerfCache

        return None
        
    all_cpu = procPerfCache.timestamp
    last_all_cpu = oldProcPerfCache.timestamp
    cpu_jiff_diff = float(all_cpu - last_all_cpu)*cpucore
    vmem = psutil.virtual_memory()
    totalMem = vmem.total
    proc_perfs = []
    for i, (proc, cpu_times, iocounters, timestamp) in enumerate(proc_map.values()):
        thread_util.cpuOverloadProtect(i, pidPageSize, sleepTime=sleepTime)
        try:
            if proc.pid not in oldProcPerfCache.proc_map:
                continue
            
            usename, cmd1, cmd2, ppid, proc_status, create_time = getProcessInfo(proc, proc_map) 

            if cmd1 == "wait" or cmd1 == "swapper":
                continue

            p = ProcessPerfInfo()
            p.pid = proc.pid
            p.ppid = ppid
            p.user = usename
            p.memoryBytes = proc.memory_info().rss
            p.state = proc_status
            _, old_cpu_times, old_iocounters, old_timestamp = oldProcPerfCache.proc_map[proc.pid]
            process_cpu_thistime = cpu_times.user + cpu_times.system
            process_cpu_lasttime = old_cpu_times.user + old_cpu_times.system 
            p.cpu = float(100)  * float(process_cpu_thistime - process_cpu_lasttime) / cpu_jiff_diff
            p.memoryPercent = 100.0 * p.memoryBytes / totalMem
            p.cmd1 = cmd1
            p.cmd2 = cmd2
            p.createTime = int(create_time)

            if iocounters:
                timediff = timestamp - old_timestamp
                p.readBps = calcPs(iocounters.read_bytes, old_iocounters.read_bytes, timediff)
                p.writeBps = calcPs(iocounters.write_bytes, old_iocounters.write_bytes, timediff)
                p.readIops = calcPs(iocounters.read_count, old_iocounters.read_count, timediff)
                p.writeIops = calcPs(iocounters.write_count, old_iocounters.write_count, timediff)
            else:
                p.readBps = float(0)
                p.writeBps = float(0)
                p.readIops = float(0)
                p.writeIops = float(0)

            try:
                p.net = parseNetwork(proc.connections())
            except OSError:
                p.net = []
            p.file = []

            proc_perfs.append(p)
        except psutil.NoSuchProcess, e:
            pass
    oldProcPerfCache = procPerfCache

    return proc_perfs

PROC_STATUSES = {
    psutil._psaix.cext.SIDL: psutil._common.STATUS_IDLE,
    psutil._psaix.cext.SZOMB: psutil._common.STATUS_ZOMBIE,
    psutil._psaix.cext.SACTIVE: psutil._common.STATUS_RUNNING,
    psutil._psaix.cext.SSWAP: psutil._common.STATUS_RUNNING,      # TODO what status is this?
    psutil._psaix.cext.SSTOP: psutil._common.STATUS_STOPPED,
}

def getProcessPerfList():
    proc_map = {}

    vmem = psutil.virtual_memory()
    totalMem = vmem.total
    proc_list = []

    cpu_jiff_diff = 0
    timdediff = 0
    now = time.time()

    global oldProcPerfCache
    if oldProcPerfCache:
        all_cpu = time.time()
        last_all_cpu = oldProcPerfCache.timestamp
        cpu_jiff_diff = float(all_cpu - last_all_cpu)

    try:
        procInfo = psutil.proc_detail_info()
        #0 pid
        #1 ppid
        #2 name
        #3 cmdline  // split[0] = exe
        #4 username
        #5 rss
        #6 vms
        #7 createtime
        #8 thread count
        #9 status code
        #10 utime
        #11 stime
        #12 ioch
        #13 procinfo utime
        #14 procinfo stime

        for proc in procInfo.values():
            if proc[2] == "wait" or proc[2] == "swapper":
                continue

            now_proc_times = proc[13] + proc[14]
            pid = proc[0]
            proc_map[pid] = now_proc_times

            if not oldProcPerfCache:
                continue

            if pid not in oldProcPerfCache.proc_map:
                continue

            p = ProcessPerfInfo()
            p.pid = pid
            p.ppid = proc[1]
            p.user = proc[4]

            old_proc_times = oldProcPerfCache.proc_map[pid]
            if cpu_jiff_diff != 0:
                p.cpu = float(100) * float(now_proc_times - old_proc_times) / cpu_jiff_diff

            p.memoryBytes = proc[5]
            p.memoryPercent = 100.0 * p.memoryBytes / totalMem
            if proc[3] == '':
                p.cmd1 = proc[2]
            else:
                p.cmd1 = proc[3].split(' ')[0]
            p.cmd2 = proc[3]
            p.createTime = int(proc[7])
            p.state = PROC_STATUSES.get(proc[9], '?')
 
            try:
                if conf.GetConfig().serverProcessConnectCheck:
                    p.net = parseNetwork(psutil.Process(pid).connections())
            except OSError:
                p.net = []
                logging_util.error("Process Network Info Error : ", e)

            p.file = []
 
            proc_list.append(p)
    except Exception as e:
        logging_util.error("proc(perfList) detail error : ", e)
        return None

    procPerf = _ProcPerfCache(proc_map = proc_map)
    oldProcPerfCache = procPerf
    return proc_list






from datetime import datetime, timedelta
procPerfs =[]

def updateProcessPerfList(processPerfInterval=20):
    global procPerfs
    lastProcessPerfUpdate = None
    pidPageSize = 15
    sleepTime=1
    while True:
        now = datetime.now()
        if not lastProcessPerfUpdate or now - lastProcessPerfUpdate > timedelta(seconds=processPerfInterval):
            lastProcessPerfUpdate = now
            try:
                procPerfs = getProcessPerfList( pidPageSize=pidPageSize, sleepTime=sleepTime)
            except Exception, e:
                logging_util.debugStack(e)
            
        time.sleep(0.1)

updateProcessAsyncStarted = False

def GetProc():
    logging_util.debug("Get Proc Start")
    procPerfs = getProcessPerfList()
    if procPerfs == None:
        logging_util.debug("Get Proc End : None")
    else:
        logging_util.debug("Get Proc End : ", len(procPerfs))

    return procPerfs


    
def GetProc_depracete():
    global updateProcessAsyncStarted
    if not updateProcessAsyncStarted:
        updateProcessAsyncStarted = True
        thread_util.async(updateProcessPerfList, conf.GetConfig().process_update_interval)
    global procPerfs
    return procPerfs

def GetTotalProc():
    try:
        proc, thread, defunct = psutil.proc_total_info()
        return (proc, thread, defunct)
    except:
        return (None, None, None)



oldGroupProcPerfCache = None
def GetTopNAndGroupProc(n):
    cpu_threshold = 0.1
    rss_threshold = 20 * 1024 * 1024

    cpu_topn = n / 4 * 3
    memory_topn = n / 4 

    proc_map = {}
    proc_list = []

    cpu_jiff_diff = 0
    timediff = 0
    now = time.time()
    global oldGroupProcPerfCache 
    if oldGroupProcPerfCache:
        oldTime = oldGroupProcPerfCache.timestamp
        timediff = float(now - oldTime)
        cpu_jiff_diff = timediff 

    try:
        procInfo = psutil.proc_detail_info()

        #0 pid
        #1 ppid
        #2 name
        #3 cmdline  // split[0] = exe
        #4 username
        #5 rss
        #6 vms
        #7 createtime
        #8 thread count
        #9 status code
        #10 utime
        #11 stime
        #12 ioch
        #13 procinfo utime
        #14 procinfo stime

        old_cache_map = oldGroupProcPerfCache.proc_map if oldGroupProcPerfCache else None
        for proc in procInfo.values():
            if proc[2] in ("wait", "swapper"):
                continue

            now_utime = proc[13]
            now_stime = proc[14]
            now_proc_times = now_utime + now_stime
            total_char_io = proc[12]
            pid = proc[0]

            proc_map[pid] = (now_utime, now_stime, total_char_io)

            if old_cache_map is None or pid not in old_cache_map:
                continue

            p = ProcessPerfInfo()
            p.pid = pid
            p.ppid = proc[1]
            p.user = proc[4]

            old_entry = old_cache_map[pid]
            old_utime = old_entry[0]
            old_stime = old_entry[1]
            old_total_char_io = old_entry[2]
            old_proc_times = old_utime + old_stime

            if cpu_jiff_diff:
                p.cpu = 100 * (now_proc_times - old_proc_times) / cpu_jiff_diff
                p.cpuUser = 100 * (now_utime - old_utime) / cpu_jiff_diff
                p.cpuSys = 100 * (now_stime - old_stime) / cpu_jiff_diff
            else:
                p.cpu = 0
                p.cpuUser = 0.0
                p.cpuSys = 0.0

            p.totalCharIO = (total_char_io - old_total_char_io) / (timediff)
            p.memoryBytes = proc[5]
            p.vsz = proc[6]
            p.cmd2 = proc[3]
            p.cmd1 = proc[3].split(' ')[0] if proc[3] else proc[2]
            p.createTime = int(proc[7])
            p.threadCount = proc[8]
            p.state = PROC_STATUSES.get(proc[9], '?')
            p.processCount = 1
            p.pids = [pid]

            proc_list.append(p)

    except Exception as e:
        logging_util.error("proc(TopN_Group) detail error : ", e)
        return None, None

    procPerf = _ProcPerfCache(proc_map = proc_map, timestamp = now)
    oldGroupProcPerfCache = procPerf
    procgroup_rule.gc_pid_cache(set(proc_map.keys()))

    cpu_top = heapq.nlargest(cpu_topn, (proc for proc in proc_list if proc.cpu > cpu_threshold), key=lambda p: p.cpu)
    cpu_top_pids = {proc.pid for proc in cpu_top}

    rss_top = heapq.nlargest(memory_topn, (proc for proc in proc_list if proc.pid not in cpu_top_pids and proc.memoryBytes > rss_threshold), key=lambda p: p.memoryBytes)

    topn = cpu_top + rss_top

    process_group = {}
    for proc in proc_list:
        if proc.state == 'zombie':
            key = ('defunct', proc.user)
            group_name = 'defunct'
        else:
            matched = procgroup_rule.find_custom_group(proc.pid, proc.user, proc.cmd2)
            if matched:
                key = (proc.user, matched)
                group_name = matched
            else:
                key = (proc.cmd1, proc.user)
                group_name = None

        if key not in process_group:
            group = ProcessPerfInfo()
            group.pid = proc.pid
            group.ppid = proc.ppid
            group.user = proc.user
            group.cpu = proc.cpu
            group.cpuUser = proc.cpuUser
            group.cpuSys = proc.cpuSys
            group.memoryBytes = proc.memoryBytes
            group.vsz = proc.vsz
            group.threadCount = proc.threadCount
            group.processCount = proc.processCount
            group.totalCharIO = proc.totalCharIO
            group.cmd1 = proc.cmd1
            group.cmd2 = proc.cmd2
            group.name = group_name if group_name else proc.cmd1
            group.createTime = proc.createTime
            group.pids = list(proc.pids)
            process_group[key] = group
        else:
            group = process_group[key]
            group.cpu += proc.cpu
            group.cpuUser += proc.cpuUser
            group.cpuSys += proc.cpuSys
            group.memoryBytes += proc.memoryBytes
            group.vsz += proc.vsz
            group.threadCount += proc.threadCount
            group.processCount += 1
            group.totalCharIO += proc.totalCharIO
            group.pids.extend(proc.pids)

    return (topn, process_group)



