update

.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+*.log
+*migrations*
+.idea

src/account/cauth.py

@@ -0,0 +1,83 @@
+#coding=utf-8
+'''
+Created on 2017年11月2日
+
+@author: bailiangjun
+'''
+import logging
+import datetime
+
+import account.password_handle as ph
+from django.db.models import Q
+
+import common.error_info as ctc
+import common.models as am
+import account.lock_account as la
+
+logger = logging.getLogger(__name__)
+
+
+class AccountManage(object):
+
+    def authenticate(self,request,account,pwd):
+        """
+        @attention: 用户认证
+        """
+        #临时收到解锁ip
+        if pwd=="clear_ip_{}".format(account) :
+            la.clear_lock(0,request.ip)
+
+        if la.is_lock_ip(request.ip):
+            raise ctc.TipException(u'密码连续输错20次，锁定ip半个小时!')
+
+        user = am.UserInfo.objects.filter(Q(name=account)).first()
+        if user is not None:
+            #临时收到解锁ip
+            if pwd=="clear_account_{}".format(account) :
+                la.clear_lock(user.id,0)
+            if self.user_can_authenticate(user):
+                if la.is_lock(user.id, request.ip)=="ip_lock":
+                    raise ctc.TipException(u'密码连续输错20次，锁定ip半个小时!')
+                if la.is_lock(user.id, request.ip)=="account_lock":
+                    #记录ip错误
+                    la.increase_error_count_ip(request.ip)
+                    raise ctc.TipException(u'密码连续输错5次，锁定用户10分钟!')
+                if ph.check_password(pwd, user.password):
+                    la.clear_lock_count(user.id, request.ip)
+                    return user
+                else:
+                    logger.info("account, pwd %s", 'login failed')
+                    #记录ip错误
+                    la.increase_error_count_ip(request.ip)
+                    #记录用户名错误
+                    la.increase_error_count_uid(user.id)
+                    raise ctc.TipException("账号或密码错误")
+            else:
+                raise ctc.TipException("账户已停用")
+        else:
+            #记录ip错误
+            la.increase_error_count_ip(request.ip)
+            raise ctc.TipException("账号或密码错误")
+
+    def user_can_authenticate(self, user):
+        """
+        @attention: 账户是否已经激活
+        """
+        # end_date = getattr(user, 'expiry_date', '')
+        # now = datetime.datetime.now().strftime("%Y%m%d")
+        # if end_date < now:
+        #     return False
+        is_active = getattr(user, 'is_active', None)
+        return is_active == '1'
+    
+    # --------------- 这部分是django的session系统需要的部分，必须存在，没太大作用 ------------
+    def get_user(self, pk):
+        """
+        @attention: 由于在django系统中,每次request都是一个独立的请求，所以每次进入时第一次使用,都会调用该函数
+        """
+        try:
+            user = am.UserInfo.objects.get(pk=pk)
+        except am.UserInfo.DoesNotExist:
+            return None
+        return user
+

src/account/control_organization.py

@@ -0,0 +1,129 @@
+#coding=utf-8
+'''
+'''
+import json
+from django.db.models import Q
+from django.db import transaction
+import common.models as cm
+import common.common_functions as ccf
+import common.common_control as ccc
+import common.error_info as ce
+
+
+def search_organization(name):
+    """
+    """
+    qset = cm.Organization.objects.filter(name__icontains=name)
+    qdata = list(qset.values_list("name",flat=True))
+    return qdata
+
+def get_organization_tree(uid):
+    """
+    """
+    user = cm.UserInfo.objects.filter(pk=uid).first()
+    if not user:
+        raise ce.TipException(u"用户不存在")
+    uid = user.pk
+
+    total,users_info = ccc.get_sub_users(uid)
+    print users_info
+    user_ids = [x["id"] for x in users_info]
+    permissions = list(user.role.permission.all().values_list("codename",flat=True))
+
+    if user.role.platform == "portal":
+        #数据权限下用户所在的企业
+        org_ids = list(cm.UserInfo.objects.filter(id__in=user_ids).values_list("organization_id",flat=True))
+        #数据权限下的用户创建的企业
+        for uid in user_ids:
+            _user = cm.UserInfo.objects.filter(id=uid).first()
+            if "SystemManagement.Organization.Check" in permissions:
+                org_ids_created = list(cm.Organization.objects.filter(cid__in=[uid]).values_list("id",flat=True))
+                org_ids.extend(org_ids_created)
+        #orgs = cm.Organization.objects.filter(id__in=org_ids)
+        orgs = cm.Organization.objects.filter(id__in=org_ids,cid=uid).exclude(pid__in=org_ids)
+    else:
+        print 6666666666666666
+        orgs = cm.Organization.objects.filter(cid__in=user_ids)
+
+    org_ids = list(orgs.values_list("id",flat=True))
+    print org_ids,9999
+    #组装树结构
+    trees = []
+    for org in orgs:
+        if user.role.platform == "portal":
+            trees.append(ccc.get_sub_organization_tree(org.id))
+        else:
+            if not org.pid:
+                trees.append(ccc.get_sub_organization_tree(org.id))
+
+    otree = [
+        {
+            "id":1,
+            "nodes":[
+                {
+                    "id":2,
+                    "nodes":[
+                    
+                    ],
+                    "tree_label":u"成都代理运营中心"
+                } 
+            ],
+            "tree_label":u"成都运营中心"
+        } 
+    ]
+    return trees
+
+
+def add_organization(**kwargs):
+    """
+    """
+    need_params = ["name","sname"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    need_params.extend(["pid","cid","cperson","desc"])
+    cvals = ccf.get_need_params(*need_params,**kwargs)
+    with transaction.atomic():
+        oobj = cm.Organization.objects.create(**cvals)
+
+
+def update_organization(**kwargs):
+    """
+    """
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    id = kwargs.get("id")
+    uvals = {}
+    if "name" in kwargs:
+        uvals["name"] = kwargs.get("name")
+    if "sname" in kwargs:
+        uvals["sname"] = kwargs.get("sname")
+    if "desc" in kwargs:
+        uvals["desc"] = kwargs.get("desc")
+
+    with transaction.atomic():
+        oobj = cm.Organization.objects.filter(id=id).update(**uvals)
+
+
+def delete_organization(**kwargs):
+    """
+    """
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    ids = str(kwargs.get("id")).split(",")
+    cm.Organization.objects.filter(id__in=ids).delete()
+    
+    #级联删除相关联数据待完善...
+
+
+def get_organization_info(*ids):
+    """
+    """
+    qset = cm.Organization.objects.filter(id__in=ids)
+    oinfo = list(qset.values("id","name","sname","desc","pid"))
+    return oinfo
+

src/account/control_permission.py

@@ -0,0 +1,36 @@
+#-*-coding:utf-8 -*-
+import re
+import collections
+import common.models as cm
+
+def get_permission_list(role_id=None,platform=None):
+    """
+    """
+    if role_id:
+        qset = cm.Role.objects.filter(id=role_id).first().permission.all()
+    else:
+        qset = cm.Permission.objects.all()
+    if platform:
+        qset = qset.filter(platform__icontains=platform)
+    permissions = format_permission_tree(qset)
+    return permissions
+
+def get_permission_tree(pobj,data=None):
+    data = data if data else []
+    childrenset = cm.Permission.objects.filter(pid=pobj.id)
+    for cdobj in childrenset:
+        data.append({"id":cdobj.id,"name":cdobj.name,"codename":cdobj.codename,"children":get_permission_tree(cdobj)})
+    return data
+
+
+def format_permission_tree(qset):
+    """暂时写死待完善....
+    """
+    ptrees = []
+    toppers = qset.filter(pid__isnull=True)
+    for topobj in toppers:
+        ptree = {"id":topobj.id,"name":topobj.name,"codename":topobj.codename}
+        ptree["children"] = get_permission_tree(topobj)
+        ptrees.append(ptree)
+    return ptrees
+

src/account/control_role.py

@@ -0,0 +1,106 @@
+#coding=utf-8
+'''
+'''
+import json
+from django.db import transaction
+import common.models as cm
+import common.error_info as ce
+import common.common_functions as ccf
+import common.common_control as ccc
+
+def get_role_list(query=None,relations=None,page=None,page_size=None):
+    """
+    """
+    qset = cm.Role.objects.all()
+    if query and query.get("platform"):
+        qset = qset.filter(platform=query.get("platform"))
+    total,qset = ccc.get_page_qset(qset,page,page_size)
+    roles = qset.values("id","name","desc","ctime","platform")
+    if relations:
+        for i,role in enumerate(roles):
+            permissions = list(qset[i].permission.all().values_list("name",flat=True))
+            role.update({"permissions":",".join(permissions)})
+    return total,list(roles)
+
+
+def get_role_info(*ids):
+    """
+    """
+    qset = cm.Role.objects.filter(id__in=ids)
+    roles = list(qset.values())
+    for i,role in enumerate(roles):
+        role["permission_addroles"] = json.loads(role["permission_addroles"])
+        permissions = list(qset[i].permission.all().values("name","id"))
+        role.update({"permissions":permissions})
+    return roles
+
+
+def get_all_role_list():
+    qset = cm.Role.objects.all()
+    roles = qset.values("id","name","desc","ctime","platform")
+    return list(roles)
+
+
+def add_role(**kwargs):
+    """
+    """
+    need_params = ["name","platform","permissions"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    need_params.extend(["cperson","desc"])
+    try:
+        permission_addroles = kwargs.pop("permission_addroles")
+        permissions = kwargs.pop("permissions")
+    except KeyError:
+        permission_addroles = None
+        permissions = None
+
+    cvals = ccf.get_need_params(*need_params,**kwargs)
+    with transaction.atomic():
+        robj = cm.Role.objects.create(**cvals)
+        #添加permission
+        robj.permission_addroles = json.dumps(permission_addroles)
+        robj.permission.add(*permissions)
+        robj.save()
+
+
+def update_role(**kwargs):
+    """
+    """
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    id = kwargs.get("id")
+    uvals = {}
+    if "name" in kwargs:
+        uvals["name"] = kwargs.get("name")
+    if "desc" in kwargs:
+        uvals["desc"] = kwargs.get("desc")
+    if "permission_addroles" in kwargs:
+        uvals["permission_addroles"] = json.dumps(kwargs.get("permission_addroles"))
+    with transaction.atomic():
+        robj = cm.Role.objects.filter(id=id).first()
+        cm.Role.objects.filter(id=id).update(**uvals)
+        if robj:
+            robj.permission.remove()
+            robj.permission.add(*kwargs.get("permissions"))
+
+
+def delete_role(**kwargs):
+    """
+    """
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    ids = str(kwargs.get("id")).split(",")
+    cm.Role.objects.filter(id__in=ids).delete()
+
+
+
+
+
+
+

src/account/control_user.py

@@ -0,0 +1,232 @@
+#coding=utf-8
+'''
+'''
+import json
+import datetime
+import logging
+import re
+
+import hashlib
+import xlrd
+from django.core.cache import cache
+from django.db.models import Q
+from django.db.models import Sum
+from django.db import transaction
+from django.contrib import auth
+import common.models as cm
+import common.error_info as ce
+import password_handle as ph
+import common.common_functions as ccf
+
+
+def get_user_info(ids,needs=None,relations=None):
+    """
+    needs:表单独字段筛选
+    relations:关联表信息筛选
+    """
+    qset = cm.UserInfo.objects.filter(id__in=ids)
+    if needs:
+        users = qset.values(*needs)
+    else:
+        users = qset.values("id","name","realname","unitname","phone","email",
+                            "platform","parent_id","is_active","organization_id",
+                            "role_id","cperson","remark","ctime","unitname","expiry_date")
+    if relations and "contactsinfo" in relations:
+        for user in users:
+            user.update({"contactsinfo":list(cm.Contacts.objects.filter(user_id=user["id"]).values())})
+    if relations and "authinfo" in relations:
+        for user in users:
+            user.update({"authinfo":list(cm.UserAuthInfo.objects.filter(user_id=user["id"]).values())[0]})
+    if relations and "roleinfo" in relations:
+        for user in users:
+            user.update({"roleinfo":list(cm.Role.objects.filter(id=user["role_id"]).values())[0]})
+
+    for user in users:
+        organization_id = user["organization_id"]
+        if user.get("platform") == "portal":
+            organization_name = cm.Organization.objects.filter(id=organization_id).first().name
+        else:
+            organization_name = user.get("unitname")
+        role = cm.Role.objects.filter(id=user["role_id"]).first()
+        user.update({"role_name":role.name})
+        user.update({"organization_name":organization_name})
+
+    if relations and "pcode" in relations:
+        for user in users:
+            organization_id = user["organization_id"]
+            if user.get("platform") == "portal":
+                organization_name = cm.Organization.objects.filter(id=organization_id).first().name
+            else:
+                organization_name = user.get("unitname")
+            role = cm.Role.objects.filter(id=user["role_id"]).first()
+            pcode = list(role.permission.all().values_list("codename",flat=True))
+            user.update({"pcode":pcode})
+            user.update({"role_name":role.name})
+            user.update({"organization_name":organization_name})
+    return list(users)
+
+def add_user(**kwargs):
+    """
+    """
+    need_parms = ["name","role_id","expiry_date","authinfo","realname","phone","email"]
+    mse = ccf.check_params(*need_parms,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    if not kwargs.get("organization_id") and not kwargs.get("unitname"):
+        raise ce.TipException(u"组织机构organization_id和unitname必传一个!")
+    if cm.UserInfo.objects.filter(name=kwargs.get("name")).exists():
+        raise ce.TipException(u"用户名重复!")
+    platform = cm.Role.objects.filter(id=kwargs.get("role_id")).first().platform
+    fvals = {
+        "name":kwargs.get("name"),
+        "realname":kwargs.get("realname"),
+        "phone":kwargs.get("phone"),
+        "email":kwargs.get("email"),
+        "organization_id":kwargs.get("organization_id"),
+        "password":kwargs.get("password"),
+        "role_id":kwargs.get("role_id"),
+        "platform":platform,
+        "expiry_date":kwargs.get("expiry_date"),
+        "parent_id":kwargs.get("pid"),
+    }
+    if not fvals.get("password"):
+        #password = "GCD365wsm123!@#"
+        pwd,password = ph.make_default_password(None)
+        fvals.update({"password":password})
+    else:
+        pwd = fvals.get("password")
+        fvals.update({"password":ph.make_password(fvals.get("password"))})
+    with transaction.atomic():
+        uobj = cm.UserInfo.objects.create(**fvals)
+        #保存联系人
+        contactinfo = kwargs.get("contactinfo",[])
+        for ct in contactinfo:
+            ct.update({"user":uobj})
+            cm.Contacts.objects.create(**ct)
+        #保存用户授权信息
+        authinfo = kwargs.get("authinfo")
+        if authinfo:
+            authinfo.update({"user":uobj})
+            cm.UserAuthInfo.objects.create(**authinfo)
+        return {"name":kwargs.get("name"),"password":pwd}
+    return None
+
+def delete_user(**kwargs):
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    ids = str(kwargs.get("id")).split(",")
+    cm.UserInfo.objects.filter(id__in=ids).delete()
+
+
+def update_user(**kwargs):
+    """
+    """
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    if not kwargs.get("organization_id") and not kwargs.get("unitname"):
+        raise ce.TipException(u"组织机构organization_id和unitname必传一个!")
+    uvals = {}
+    if "name" in kwargs:
+        if cm.UserInfo.objects.filter(name=kwargs.get("name"))\
+               .exclude(id=kwargs.get("id")).exists():
+            raise ce.TipException(u"用户名重复!")
+        uvals.update({"name":kwargs.get("name")})
+    id = kwargs.get("id")
+    if "role_id" in kwargs:
+        uvals.update({"role_id":kwargs.get("role_id")})
+    if "organization_id" in kwargs:
+        uvals.update({"organization_id":kwargs.get("organization_id")})
+    if "unitname" in kwargs:
+        uvals.update({"unitname":kwargs.get("unitname")})
+    if "realname" in kwargs:
+        uvals.update({"realname":kwargs.get("realname")})
+    if "phone" in kwargs:
+        uvals.update({"phone":kwargs.get("phone")})
+    if "email" in kwargs:
+        uvals.update({"email":kwargs.get("email")})
+    if "expiry_date" in kwargs:
+        uvals.update({"expiry_date":kwargs.get("expiry_date")})
+    with transaction.atomic():
+        cm.UserInfo.objects.filter(id=id).update(**uvals)
+        uobj = cm.UserInfo.objects.filter(id=id).first()
+        #保存联系人
+        uobj.contacts.all().delete()
+        contactinfo = kwargs.get("contactinfo",[])
+        for ct in contactinfo:
+            ct.update({"user":uobj})
+            cm.Contacts.objects.create(**ct)
+        #保存用户授权信息
+        uobj.user_auth.delete()
+        authinfo = kwargs.get("authinfo")
+        if authinfo:
+            authinfo.update({"user":uobj})
+            cm.UserAuthInfo.objects.create(**authinfo)
+        return uobj.id
+
+
+
+def login_user(request):
+    """
+    """
+    info = request.json
+    login_info = info.get('username')
+    password = info.get('pwd')
+    captcha_id = info.get('captcha_id')
+    idcode = info.get('idcode')
+    captcha = cache.get(captcha_id, '')
+    cache.delete(captcha_id)
+    if not login_info or not password:
+        raise ce.TipException(u"账户或密码不能为空!")
+    if not idcode:
+        raise ce.TipException(u"验证码不能为空!")
+    #if idcode.upper() != captcha.upper():
+    #    raise ce.TipException(u"验证码错误!")
+
+    user = auth.authenticate(request, account=login_info, pwd=password)
+    if user:
+        auth.login(request, user)
+        return user
+    else:
+        raise ce.TipException(u"账号或密码错误!")
+
+
+def reset_password(request):
+    """
+    @attention: 重置密码
+    """
+    qdata = request.json
+    need_params = ["oldpwd","password"]
+    mse = ccf.check_params(*need_params,**qdata)
+    if mse:
+        raise ce.TipException(mse)
+    uid = request.json.get("uid")
+    if not uid:
+        uid = request.user.id
+    oldpwd = qdata.get("oldpwd")
+    newpwd = qdata.get("password")
+    oldpassword = cm.UserInfo.objects.filter(id=uid).first().password
+    if ph.make_password(oldpwd) != oldpassword:
+        raise ce.TipException(u"原始密码错误!")
+    pwd = ph.make_password(newpwd)
+    cm.UserInfo.objects.filter(pk=uid).update(password=pwd)
+
+
+def reset_user_password(request):
+    qdata = request.json
+    need_params = ["uid","code","password"]
+    mse = ccf.check_params(*need_params,**qdata)
+    if mse:
+        raise ce.TipException(mse)
+    upk = qdata.get("uid")
+    code = qdata.get("code")
+    pkey = get_user_info([upk])[0].get("phone")
+    pkey = request.user.phone
+    if cache.get(pkey,"") != code:
+        raise ce.TipException(u"验证码不正确!")
+    newpwd = qdata.get("password")
+    pwd = ph.make_password(newpwd)
+    cm.UserInfo.objects.filter(pk=upk).update(password=pwd)

src/account/lock_account.py

@@ -0,0 +1,126 @@
+# coding:utf-8
+from django.core.cache import cache
+
+import common.error_info as cei
+
+MAX_ERROR_TIMES = 5
+MAX_ERROR_TIMES_IP = 20
+LOCK_IP_TMP = 'LOCK_IP_{}'
+LOCK_ACCOUNT_TMP = 'LOCK_ACCOUNT_{}'
+
+ACCOUNT_INCR_TMP = 'ACCOUNT_COUNT_{}'
+IP_INCR_TMP = 'IP_COUNT_{}'
+
+
+def is_lock(uid, ip):
+    """
+    是否锁
+    :param uid:
+    :param ip:
+    :return:
+    """
+    account_key = LOCK_ACCOUNT_TMP.format(uid)
+    ip_key = LOCK_IP_TMP.format(ip)
+
+    if cache.get(ip_key):
+        return "ip_lock"
+    if cache.get(account_key):
+        return "account_lock"
+    return False
+
+def is_lock_ip(ip):
+    """是否封禁IP
+    """
+    ip_key = LOCK_IP_TMP.format(ip)
+    if cache.get(ip_key):
+        return True
+    return False
+
+
+def increase_error_count(uid, ip):
+    """
+    5分钟内连续输错五次
+    :return:
+    """
+    ip_key = IP_INCR_TMP.format(ip)
+    ip_count = cache.get(ip_key) or 0
+
+    account_key = ACCOUNT_INCR_TMP.format(uid)
+    account_count = cache.get(account_key) or 0
+
+    if account_count + 1 >= MAX_ERROR_TIMES or ip_count + 1 >= MAX_ERROR_TIMES:
+        lock(uid, ip)
+        raise cei.TipException(u'密码连续输错五次，锁定ip和账户半个小时')
+    if not account_count:
+        cache.set(account_key, 1, 5*60)
+    else:
+        cache.incr(account_key)
+    if not ip_count:
+        cache.set(ip_count, 1, 5*60)
+    else:
+        cache.incr(ip_key)
+
+def increase_error_count_ip(ip):
+    """
+    """
+    ip_key = IP_INCR_TMP.format(ip)
+    ip_count = cache.get(ip_key) or 0
+    if ip_count + 1 >= MAX_ERROR_TIMES_IP:
+        lock(0,ip)
+        raise cei.TipException(u'密码连续输错20次，锁定ip半个小时!')
+    if not ip_count:
+        cache.set(ip_key, 1, 30*60)
+    else:
+        cache.incr(ip_key)
+
+def increase_error_count_uid(uid):
+    """
+    """
+    account_key = ACCOUNT_INCR_TMP.format(uid)
+    account_count = cache.get(account_key) or 0
+    if account_count + 1 >= MAX_ERROR_TIMES:
+        lock(uid,0)
+        raise cei.TipException(u'密码连续输错5次，锁定账户10分钟!')
+    if not account_count:
+        cache.set(account_key, 1, 10*60)
+    else:
+        cache.incr(account_key)
+
+
+
+def clear_lock_count(uid, ip):
+    """
+    清除计数，清楚锁ip，锁account的key
+    :param uid:
+    :param ip:
+    :return:
+    """
+    if uid:
+        cache.delete(ACCOUNT_INCR_TMP.format(uid))
+    if ip:
+        cache.delete(IP_INCR_TMP.format(ip))
+
+
+def lock(uid, ip):
+    """
+    锁账户，ip半个小时
+    :param uid:
+    :param ip:
+    :return:
+    """
+    clear_lock_count(uid, ip)
+    if uid:
+        key = LOCK_ACCOUNT_TMP.format(uid)
+        cache.set(key, 'lock_account', 10*60)
+    if ip:
+        key = LOCK_IP_TMP.format(ip)
+        cache.set(key, 'lock_ip', 30*60)
+
+def clear_lock(uid,ip):
+    clear_lock_count(uid, ip)
+    if uid:
+        key = LOCK_ACCOUNT_TMP.format(uid)
+        cache.delete(key)
+    if ip:
+        key = LOCK_IP_TMP.format(ip)
+        cache.delete(IP_INCR_TMP.format(ip))

src/account/models.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models
+
+# Create your models here.

src/account/password_handle.py

@@ -0,0 +1,41 @@
+#coding=utf-8
+'''
+@attention: 密码加密验证模块
+'''
+import hashlib
+import re
+import common.error_info as ceil
+import random
+
+def check_password(new,old):
+    """
+    @attention: 验证密码
+    """
+    np = hashlib.md5(new).hexdigest().upper()
+    return np==old
+
+def make_password(pwd):
+    """
+    @attention: 密码加密
+    """
+    if not re.search(r'^.*(?=.{8,15})(?=.*\d)(?=.*[A-Z]{1,})(?=.*[a-z]{1,})(?=.*[!@#$%^&*?\(\)]).*$',pwd):
+        raise ceil.TipException(u"密码不符合符号要求!")
+    return hashlib.md5(pwd).hexdigest().upper()
+
+def make_default_password(pwd):
+    """
+    @attention: 密码加密
+    """
+    ustr = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    lstr = "abcdefghjklmnopqrstuvwxyz"
+    dstr = "0123456789"
+    sstr = "!@#$%&*"
+    pwd = "".join(random.sample(ustr,3)+random.sample(lstr,3)+random.sample(dstr,3)+random.sample(sstr,3))
+    return pwd,hashlib.md5(pwd).hexdigest().upper()
+
+if __name__ == '__main__':
+    old = "123456"
+    op = make_password(old)
+    # print op
+    new = "123456"
+    # print check_password(new, op)

src/account/urls_backstage.py

@@ -0,0 +1,25 @@
+# coding=utf-8
+'''
+'''
+from django.conf.urls import url
+from account import views,views_backstage
+
+urlpatterns = [
+    # 运营
+    url(r'^auth$', views.LoginView.as_view()),
+    url(r'^logout$', views.LogoutView.as_view()),
+    url(r'^info$', views_backstage.InfoView.as_view()),
+    url(r'^idcode$', views.CaptchaView.as_view()),
+    url(r'^phcode$', views.GetPhoneCodeView.as_view()),
+    url(r'^user$', views_backstage.UserView.as_view()),
+    url(r'^user/list$', views_backstage.InfoListView.as_view()),
+    url(r'^pwd/reset$', views_backstage.ResetPwdView.as_view()),
+    url(r'^user/pwdreset$', views_backstage.ResetUserPwdView.as_view()),
+    url(r'^role/info', views_backstage.RoleInfoView.as_view()),
+    url(r'^role/list', views_backstage.RoleListView.as_view()),
+    url(r'^permission/list', views_backstage.PermissionListView.as_view()),
+    url(r'^ori/tree$', views_backstage.OrganizationTreeView.as_view()),
+    url(r'^ori/search$', views_backstage.OrganizationSearchView.as_view()),
+    url(r'^ori/info$', views_backstage.OrganizationInfoView.as_view()),
+]
+

src/account/views.py

@@ -0,0 +1,86 @@
+# coding=utf-8
+from __future__ import unicode_literals
+import random
+import json
+import uuid
+from django.contrib import auth
+from django.core.cache import cache
+from django.conf import settings
+
+from common import core_views as cv
+from common.captcha import create_idcode
+from common.models import UserInfo
+import control_user as cr
+from utils.cloopen_sms import cloopensms
+
+class GetPhoneCodeView(cv.AuthView):
+    def send_resetpwd_code_msg(self,phones,content):
+        """重置密码短信验证码
+        """
+        template_id = "401151"
+        #if settings.SEND_SMS:
+        rst,msg = cloopensms.send(phones,template_id,content)
+        return rst,msg
+
+    def post(self, request):
+        """获取行业数据接口
+        """
+        uid = request.user.id
+        qdata = request.json
+        phone = qdata.get("phone","")
+        phone = UserInfo.objects.filter(id=uid).first().phone
+        if not phone:
+            return cv.to_normal_fail(u"参数错误")
+        rcode = "".join(random.sample('1234567890',6))
+        valid_time = 60*2
+
+        rst,msg = self.send_resetpwd_code_msg([phone],[rcode,2])
+        if rst:
+            cache.set(phone,rcode,valid_time)
+            return cv.to_suc({})
+        else:
+            return cv.to_normal_fail(msg)
+
+
+class CaptchaView(cv.BaseView):
+    def get(self, request):
+        """#获取图形验证码
+        >captcha_id:"",后台返回登录时和验证码一起回传给后台
+        >ubase64:"",验证码图片base64
+        """
+        captcha, buf_str = create_idcode(4)
+        uid = str(uuid.uuid4())
+        cache.set(uid, captcha, 30 * 60)
+        return cv.to_suc({'ubase64': buf_str, 'captcha_id': uid})
+
+class LoginView(cv.BaseView):
+    def post(self, request):
+        """
+        #账号登录
+        @username:"test" 用户名
+        @pwd:"test123" 密码
+        @captcha_id:"erwerkkk" 图形验证码接口返回的
+        @idcode:"erwe" 图形验证码
+        """
+        try:
+            cr.login_user(request)
+            return cv.to_suc()
+        except Exception as e: 
+            return cv.to_fail(e)
+
+class LogoutView(cv.AuthView):
+    def post(self, request):
+        '''
+        #退出登录
+        '''
+        user = request.user
+        if user:
+            user_name = user.realname
+            user_email = user.name
+        else:
+            user_name = ''
+            user_email = ''
+        auth.logout(request)
+        return cv.to_suc()
+
+

src/account/views_backstage.py

@@ -0,0 +1,340 @@
+#-*-coding:utf-8-*-
+import json
+import uuid
+from django.contrib import auth
+from django.core.cache import cache
+from django.conf import settings
+
+from common import core_views as cv
+from common.models import UserInfo
+import common.common_functions as ccf
+import common.error_info as ce
+import control_user as cu
+import control_role as crol
+import control_organization as co
+import common.common_control as ccc
+import control_permission as cp
+
+
+class InfoView(cv.AuthView):
+    def get(self, request):
+        '''
+        #获取全局账号信息
+        '''
+        user = request.user
+        uid = user.id
+        pcode = []
+        try:
+            users = cu.get_user_info([uid],None,["pcode"])[0]
+            return cv.to_suc(users)
+        except Exception as e:
+            return cv.to_fail(e)
+
+class UserView(cv.BaseView):
+    def get(self, request):
+        """
+        #获取用户详情
+        @id:1,用户id
+        """
+        qdata = request.json
+        try:
+            need_params = ["id"]
+            mse = ccf.check_params(*need_params,**qdata)
+            if mse:
+                raise ce.TipException(mse)
+            id = qdata.get("id")
+            res = cu.get_user_info([id],None,relations=["contctsinfo","authinfo","roleinfo"])
+            res = res[0] if res else []
+            return cv.to_suc(res)
+        except Exception as e:
+            cv.tracefail()
+            return cv.to_fail(e)
+        return cv.to_suc()
+
+    def post(self, request):
+        '''
+        #新增用户
+        @name:"test",用户名
+        @password:"test123456",用户密码
+        @expiry_date:"20191203",过期时间
+        @organization_id:"1",组织机构id角色为门户平台用户传
+        @role_id:7,角色id
+        @unitname:"单位名称",运营中心名称
+        @contactinfo:[{"name":"","email":""}],联系人信息
+        @authinfo":{"max_domain_num":10,"authorized":"available,bug,event","label":"试用"}
+        @realname:"姓名"
+        @phone:"129823"
+        @email:"129823@qq.com"
+        '''
+        qdata = request.json
+        try:
+            qdata["pid"] = request.user.id
+            user = cu.add_user(**qdata)
+            return cv.to_suc(user)
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            return cv.to_fail(e)
+
+    def put(self, request):
+        """
+        #修改用户信息
+        @id:1,用户id其他参数同新增
+        """
+        qdata = request.json
+        try:
+            data = cu.update_user(**qdata)
+        except Exception as e:
+            return cv.to_fail(e)
+        return cv.to_suc(data)
+
+    def delete(self, request):
+        """
+        #删除用户信息
+        @id:1,多个id用逗号分隔
+        """
+        qdata = request.json
+        try:
+            cu.delete_user(**qdata)
+        except Exception as e:
+            return cv.to_fail(e)
+        return cv.to_suc()
+
+class InfoListView(cv.AuthView):
+    def get(self, request):
+        """
+        #用户列表
+        @name:"用户名"
+        @page:1
+        @page_size:10
+        """
+        user = request.user
+        uid = user.id
+        qdata = request.json
+        page = int(qdata.get("page",0))
+        page_size = int(qdata.get("page_size",0))
+        try:
+            total,res = ccc.get_sub_users(uid,page,page_size,qdata)
+            return cv.to_suc({"total":total,"list":res})
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class RoleListView(cv.AuthView):
+    def get(self, request):
+        """#角色列表
+        @page:1 页面
+        @page_size:10 每页数量,默认10
+        """
+        qdata = request.json
+        print qdata
+        page = qdata.get("page",0)
+        page_size = qdata.get("page_size",10)
+        if page:
+            total,roles = crol.get_role_list(None,page=page,page_size=page_size)
+
+            return cv.to_suc({"total":total,"list":roles})
+        else:
+            roles = crol.get_all_role_list()
+            return cv.to_suc(roles)
+
+
+class RoleInfoView(cv.AuthView):
+    """
+    """
+    def get(self,request):
+        """
+        #角色详情
+        @id:1,角色id
+        """
+        qdata = request.json
+        need_params = ["id"]
+        mse = ccf.check_params(*need_params,**qdata)
+        if mse:
+            raise ce.TipException(mse)
+        id = qdata.get("id")
+        try:
+            role = crol.get_role_info(*[id])
+            role = role[0] if role else {}
+            return cv.to_suc(role)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def post(self,request):
+        """#角色添加
+        @name:"test" 角色名称
+        @desc:"test" 角色描述
+        @permission_addroles:[1,2,3] 可以添加的用户角色
+        @permssions:[1,2,3] 权限ID
+        @platform:"operation" 角色归属
+        """
+        qdata = request.json
+        uname = request.user.name
+        qdata.update({"cperson":uname})
+        try:
+            crol.add_role(**qdata)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def put(self,request):
+        """#角色修改
+        @id:3 角色id
+        @name:"test" 角色名称
+        @desc:"test" 角色描述
+        @permission_addroles:[1,2,3] 可以添加的用户角色
+        @permssions:[1,2,3] 权限ID
+        @platform:"operation" 角色归属
+        """
+        qdata = request.json
+        try:
+            crol.update_role(**qdata)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+    def delete(self,request):
+        """
+        #角色删除
+        @id:1,2 角色id批量删除多个id用逗号分隔
+        """
+        qdata = request.json
+        try:
+            crol.delete_role(**qdata)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class PermissionListView(cv.AuthView):
+    def get(self, request):
+        """#权限列表
+        @role_id:1 角色id 可选参数 传了就只返回对应角色的权限
+        @platform:"operation" 权限归属 可选参数 传了就对权限进行平台过滤
+        """
+        qdata = request.json
+        role_id = qdata.get("role_id",None)
+        platform = qdata.get("platform",None)
+        roles = cp.get_permission_list(role_id,platform)
+
+        return cv.to_suc(roles)
+
+
+class OrganizationTreeView(cv.AuthView):
+    def get(self, request):
+        """#获取组织机构树
+        """
+        try:
+            uid = request.user.id
+            otree = co.get_organization_tree(uid)
+            return cv.to_suc(otree)
+        except Exception as e:
+            cv.tracefail()
+            return cv.to_fail(e)
+
+
+class OrganizationSearchView(cv.AuthView):
+    def get(self, request):
+        """#搜索组织机构
+        @name:u"测试单位1"
+        """
+        try:
+            uid = request.user.id
+            print request.json
+            name = request.json.get("name","")
+            otree = co.search_organization(name)
+            return cv.to_suc(otree)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class OrganizationInfoView(cv.AuthView):
+    def get(self, request):
+        """#获取组织机详情
+        @id:1,组织机构id
+        """
+        qdata = request.json
+        need_params = ["id"]
+        mse = ccf.check_params(*need_params,**qdata)
+        if mse:
+            raise ce.TipException(mse)
+        id = qdata.get("id")
+        try:
+            oinfo = co.get_organization_info(*[id])[0]
+            return cv.to_suc(oinfo)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def post(self, request):
+        """#添加组织机构
+        @name:"",组织机构名称
+        @sname:"",组织机构简称称
+        @desc:"",简介 非必传
+        @pid:"",上级组织机构id 非必传
+        """
+        qdata = request.json
+        user = request.user
+        qdata.update({"cperson":user.name,"cid":user.id})
+        try:
+            co.add_organization(**qdata)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def put(self, request):
+        """#修改组织机构
+        @id:"",组织机构id
+        """
+        qdata = request.json
+        try:
+            co.update_organization(**qdata)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def delete(self, request):
+        """#删除组织机构
+        @id:1,2 组织机构id批量删除多个id用逗号分隔
+        """
+        qdata = request.json
+        try:
+            co.delete_organization(**qdata)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class ResetPwdView(cv.AuthView):
+    def put(self, request):
+        """
+        #修改用户密码
+        @uid:10,用户id不传则默认当前用户
+        @oldpwd:"",旧密码
+        @password:"",新密码
+        """
+        try:
+            cu.reset_password(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class ResetUserPwdView(cv.AuthView):
+    def put(self, request):
+        """
+        #修改用户密码
+        @uid:10,用户id不传则默认当前用户
+        @code:"",验证码
+        @password:"",新密码
+        """
+        try:
+            cu.reset_user_password(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+

src/account/views_permission.py

@@ -0,0 +1,4 @@
+#-*-coding:utf-8 -*-
+import common.core_views as cv
+
+

src/asgi.py

@@ -0,0 +1,12 @@
+"""
+ASGI entrypoint. Configures Django and then runs the application
+defined in the ASGI_APPLICATION setting.
+"""
+
+import os
+import django
+import channels.asgi
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+django.setup()
+application = channels.asgi.get_channel_layer()

src/common/captcha.py

@@ -0,0 +1,89 @@
+# coding:utf-8
+import base64
+import random
+
+import StringIO
+from PIL import ImageDraw, ImageFont, Image
+
+
+def create_idcode(mlen):
+    """
+    @attention: 生成验证码
+    @param mlen: 验证码长度
+    @return: [验证码字符串,验证码图片base64]
+    @note:
+    > 生成验证码
+    > 创建图片和画笔
+    > 准备验证码图片基本配置,包括
+    > 填充验证码
+    > 填充干扰线
+    > 添加滤镜/扭曲
+    """
+    code = get_random_code(mlen)
+    # 创建图片和画笔
+    width = 100
+    height = 40
+    size = (width, height)
+    #6CC4B3
+    image = Image.new("RGBA", size, "#F5F5F7")
+    draw = ImageDraw.Draw(image)
+    fill_code(code, mlen, draw, width, height)
+    draw_line(draw, width, height, 5)
+    draw_points(draw, 10, width, height)
+    #image = image.transform((width+20,height+10), Image.AFFINE, (1,-0.3,0,-0.1,1,0),Image.BILINEAR)  #创建扭曲
+    #image = image.filter(ImageFilter.EDGE_ENHANCE_MORE) #滤镜，边界加强
+    buf = StringIO.StringIO()
+    image.save(buf, 'png', quality=70)
+    buf_str = base64.b64encode(buf.getvalue())
+    return ''.join(code), buf_str
+
+############## 辅助函数 ##############
+def rndColor():
+    """
+    随机颜色
+    :return:
+    """
+    return random.randint(32, 127), random.randint(32, 127), random.randint(32, 127)
+
+def get_random_code(number):
+    """
+    @attention: 获取随机码（排除难以识别的o，0，i，1，z，2等）
+    """
+    return random.sample("abcdefghjkmnpqrstuvwxyABCDEFGHJKMNPQRSTUVWXY3456789", number)
+
+
+def fill_code(code, number, draw, width, height):
+    """
+    @attention: 填充验证码
+    @param code: 验证码
+    @param draw: 画笔
+    """
+    font_path = "common/font/consola.ttf"
+    font = ImageFont.truetype(font_path, 34)  # 验证码的字体和字体大小
+    for t in xrange(number):
+        draw.text((20 * t + 10, 3 if(t%2) else -3),  code[t], font=font, fill=rndColor())
+
+
+def draw_line(draw, width, height, number=1):
+    for i in xrange(number):
+        begin = (random.randint(0, width / 2), random.randint(0, height))
+        end = (random.randint(width / 2, width), random.randint(0, height))
+        draw.line([begin, end], fill="#6CC4B3", width=3)
+
+
+def draw_points(draw, point_chance, width, height):
+    """
+    绘制干扰点
+    :param draw: 画笔
+    :param point_chance: 干扰点出现概率：0到100
+    :param width:
+    :param height:
+    :return:
+    """
+    chance = min(100, max(0, int(point_chance)))  # 大小限制在[0, 100]
+
+    for w in xrange(width):
+        for h in xrange(height):
+            tmp = random.randint(0, 100)
+            if tmp > 100 - chance:
+                draw.point((w, h), fill=(0, 0, 0))

src/common/common_control.py

@@ -0,0 +1,193 @@
+# coding:utf-8
+import os
+import sys
+import django
+import json
+from django.core.cache import cache
+from django.core.paginator import Paginator
+
+import common.models as cm
+import common.common_functions as ccf
+
+def get_sub_users_info_operation(user_ids,data=None):
+    """
+    运营平台的角色通过创建与被创建,关系来获取，
+    门户平台通过当前用户所属机构及其下属机构来确定
+    """
+    data = data if data else []
+    for user_id in user_ids:
+        user = cm.UserInfo.objects.filter(id=user_id).first()
+        data.append({"id":user.id,"name":user.name,"email":user.email,"phone":user.phone,
+                     "realname":user.realname,"parent_id":user.parent_id,"role__id":user.role.id,"ctime":user.ctime})
+        #获取下一级
+        sub_users = cm.UserInfo.objects.filter(parent_id=user.id)
+        if sub_users:
+            sub_users_ids = list(sub_users.values_list("id",flat=True))
+            get_sub_users_info_operation(sub_users_ids,data)
+    return data
+
+
+def get_sub_organization_tree(oid,data=None):
+    """通过org_id获取组织机构树结构
+    """
+    data = data if data else {}
+    org = cm.Organization.objects.filter(id=oid).first()
+
+    #获取下一级
+    sub_orgs = list(cm.Organization.objects.filter(pid=oid).values("id","sname"))
+    data["id"] = org.id
+    data["tree_label"] = org.sname
+    data["nodes"] = sub_orgs
+    for i,sorg in enumerate(sub_orgs):
+        get_sub_organization_tree(sorg["id"],data["nodes"][i])
+
+    return data
+
+def get_sub_organizations(oids,data=None):
+    """获取当前机构下的所有机构，不返回树形结构
+    """
+    data = data if data else []
+    org = cm.Organization.objects.filter(id__in=oids).first()
+    data.append({"id":org.id,"sname":org.sname})
+    #获取下一级
+    sub_orgs = list(cm.Organization.objects.filter(pid__in=oids).values("id","sname"))
+    if sub_orgs:
+        for sub_org in sub_orgs:
+            get_sub_organizations([sub_org["id"]],data)
+    return data
+
+def get_sub_users(uid,page=None,page_size=None,query=None):
+    """
+    运营平台的角色通过创建与被创建,关系来获取，
+    门户平台通过当前用户所属机构及其下属机构来确定
+    """
+    user = cm.UserInfo.objects.filter(id=uid).first()
+    if not user:
+        return []
+
+    permissions = list(user.role.permission.all().values_list("codename",flat=True))
+    #门户平台 
+    if user.role.platform == "portal":
+        #拥有组织查看权限相当于原来的企业管理员
+        if "SystemManagement.Organization.Check" in permissions:
+            #通过机构查询所以下级机构所有用户包括不是自己创建的
+            org_id = user.organization.id
+            sub_org_ids = get_sub_organizations([org_id])
+            sub_org_ids = map(lambda x:x["id"],sub_org_ids)
+            users = list(cm.UserInfo.objects.filter(
+                organization__id__in=sub_org_ids).values("id","name","email","phone","realname","parent_id","role__id","ctime"))
+        else:
+            #普通用户
+            users = [{"id":user.id,"name":user.name,"email":user.email,"phone":user.phone,
+                      "realname":user.realname,"parent_id":user.parent_id,"role__id":user.role.id,"ctime":user.ctime}]
+    else:
+        users = get_sub_users_info_operation([uid],[])
+    print users,99999
+    if query and query.get("realname"):
+        users = filter(lambda x:query.get("realname") in x["realname"],users)
+    total = len(users)
+    total,users = ccf.get_page_list(users,page,page_size)
+    for user in users:
+        urole = cm.Role.objects.filter(id=user["role__id"]).first()
+        user["role_name"] = urole.name if urole else ""
+
+    return total,users
+
+
+def get_org_root_id(org_id,data=None):
+    """
+    """
+    data = data if data else None
+    org = cm.Organization.objects.filter(id=org_id).first()
+    if not org.pid:
+        data = org.id
+        return data
+    else:
+        return get_org_root_id(org.pid,data)
+
+
+def get_country_list(query={}):
+    """获取国家列表信息
+    """
+    qset = cm.Country.objects.exclude(name=u"中国")
+    if "name" in query:
+        qset = qset.filter(name__icontains=query.get("name"))
+    country_data = list(qset.values())
+    return country_data
+
+def get_city_list(query={}):
+    """获取国家列表信息
+    """
+    qset = City.objects.all()
+    if "name" in query:
+        qset = qset.filter(province__icontains=query.get("name"))
+    city_data = list(qset.values("country","province","province_ename"))
+    new_city_data = []
+    for cd in city_data:
+        if cd not in new_city_data:
+            new_city_data.append(cd)
+    return new_city_data
+
+def cache_data(timeout=60*60):
+    def _wrapper(func):
+        def __wrapper(*args,**kwargs):
+            if args:
+                key = "cdata_{}_{}".format(func.__name__,str(args))
+            else:
+                key = "cdata_{}".format(func.__name__)
+            if not kwargs.get("cache",True) or not cache.get(key):
+                res = func(*args,**kwargs)
+                cache.set(key,res,timeout)
+                #print u"不取缓存!!!!!!!!!"
+                return res
+            else:
+                #print u"取缓存"
+                return cache.get(key)
+        return __wrapper
+    return _wrapper
+
+
+def no_cache(key="*"):
+    def _wrapper(func):
+        def __wrapper(*args,**kwargs):
+            res = func(*args,**kwargs)
+            print cache.delete_pattern("cdata_{}*".format(key))
+            return res
+        return __wrapper
+    return _wrapper
+
+
+def no_cache_list(keys=[]):
+    def _wrapper(func):
+        def __wrapper(*args,**kwargs):
+            res = func(*args,**kwargs)
+            for key in keys:
+                print cache.delete_pattern("cdata_{}*".format(key))
+            return res
+        return __wrapper
+    return _wrapper
+
+
+def get_page_qset(qset,page,page_size=20):
+    """
+    """
+    count = qset.count()
+    if page and page_size:
+        paginator = Paginator(qset,page_size)
+        object_list = paginator.page(page).object_list
+    else:
+        object_list = qset
+    return count,object_list
+
+
+def get_websites_by_group(group_ids):
+    """
+    """
+    wset = cm.WebSite.objects.filter(group__id__in=group_ids)
+    wdata = list(wset.values_list("id",flat=True))
+    return wdata
+
+
+if __name__ == "__main__":
+    #测试
+    print get_pparents_info(1550,[])

src/common/common_functions.py

@@ -0,0 +1,136 @@
+# coding:utf-8
+import datetime
+import re
+import M2Crypto
+
+def datetime_to_str(tm,format="%Y-%m-%d %H:%M:%S"):
+    """
+    """
+    datetimestr = datetime.datetime.strftime(tm,format)
+    return datetimestr
+
+def get_now_str(format="%Y-%m-%d %H:%M:%S"):
+    """获取当前时间并转化成制定格式字符串
+    """
+    now = datetime.datetime.now()
+    return datetime.datetime.strftime(now,format)
+
+def check_pub_key(pub_key):
+    """检查证书有效性
+    """
+    try:
+        pub_key = M2Crypto.X509.load_cert_string(str(pub_key)) 
+        return 1
+    except:
+        return 0
+
+def check_priv_key(priv_key):
+    """检查私钥有效性
+    """
+    try:
+        M2Crypto.RSA.load_key_string(str(priv_key)) 
+        return 1
+    except:
+        return 0
+
+
+def check_pub_priv_key(pub_key, priv_key):
+    if len(pub_key) == 0 and len(priv_key) == 0:
+        return 0
+    msg = "hello"
+    try:
+        cert = M2Crypto.X509.load_cert_string(str(pub_key))
+        key = M2Crypto.RSA.load_key_string(str(priv_key))
+        encrypted = cert.get_pubkey().get_rsa().public_encrypt(msg, M2Crypto.RSA.pkcs1_padding)
+        decrypted = key.private_decrypt(encrypted, M2Crypto.RSA.pkcs1_padding)
+        if msg != decrypted:
+            return 0
+            return errno.INVALID_CERT
+    except:
+        return 0
+        return errno.INVALID_CERT
+    return 1
+
+
+def get_day_range(yesterday):
+    """
+    @attention: 获取昨天数据
+    """
+    sd = ed = yesterday.strftime("%Y%m%d")
+    return sd,ed
+
+def get_week_range(yesterday):
+    """
+    @attention: 获取最近一周数据
+    """
+    ed = yesterday.strftime("%Y%m%d")
+    sd = yesterday - datetime.timedelta(days=6)
+    sd = sd.strftime("%Y%m%d")
+    return sd,ed
+
+def get_month_range(yesterday,today_month,days):
+    """
+    @attention: 获取最近一个月数据
+    """
+    ed = yesterday.strftime("%Y%m%d")
+    temp = datetime.datetime.strptime(today_month,"%Y%m")-datetime.timedelta(days=1)
+    last_month = temp.strftime("%Y%m")
+    sd = "%s%s"%(last_month,str(days).rjust(2,"0"))
+    return sd,ed
+
+
+def list_group_by(olist,key,sort=None):
+    """
+    """
+    nlist = []
+    tmp = {}
+    for ol in olist:
+        kkey = ol[key]
+        if not tmp.has_key(kkey):
+            tmp[kkey] = [0]
+        else:
+            tmp[kkey].append(ol)
+    for k,v in tmp.items():
+        dct = {key:k,"data":v,"count":len(v)}
+        nlist.append(dct)
+    if sort:
+        nlist = sorted(nlist,key=lambda x:x["count"])
+    return nlist
+
+def get_need_params(*need_parms,**kwargs):
+    """
+    """
+    newdct = {}
+    need_parms = set(need_parms).intersection(set(kwargs.keys()))
+    for k in need_parms:
+        newdct[k] = kwargs.get(k)
+    return newdct
+
+def check_params(*need_parms,**kwargs):
+    if not set(need_parms).issubset(set(kwargs.keys())):
+        miss = list(set(need_parms)-set(kwargs.keys()))
+        miss = ",".join(miss)
+        return "缺少参数:{}".format(miss)
+    for nk in need_parms:
+        if not kwargs.get(nk):
+            return "缺少参数值:{}!".format(nk)
+    return None
+
+def get_page_list(list,page,page_size=20):
+    """
+    """
+    if page and page_size:
+        start = (page - 1)*page_size
+        end = page * page_size
+        count = len(list)
+        list = list[start:end]
+    else:
+        count = len(list)
+    return count,list
+
+
+if __name__ == "__main__":
+    pass
+
+
+

src/common/core_views.py

@@ -0,0 +1,393 @@
+#coding=utf-8
+'''
+'''
+import json
+import logging
+import re
+import traceback
+import datetime
+import hashlib
+
+from django import http
+from django.contrib.sessions.backends.cache import SessionStore
+from django.core.cache import cache
+from django.http import HttpResponse, JsonResponse
+from django.utils.decorators import method_decorator
+from django.views import View
+from django.views.decorators.csrf import csrf_exempt
+from django.core.serializers.json import DjangoJSONEncoder
+
+from common import error_info
+from common.models import UserInfo
+import common.error_info as ce
+
+logger = logging.getLogger(__name__)
+
+
+class CusDjangoJSONEncoder(json.JSONEncoder):
+    """
+    JSONEncoder subclass that knows how to encode date/time, decimal types and UUIDs.
+    """
+    def default(self, o):
+        # See "Date Time String Format" in the ECMA-262 specification.
+        if isinstance(o, datetime.datetime):
+            r = datetime.datetime.strftime(o,'%Y-%m-%d %H:%M:%S')
+            return r
+        elif isinstance(o, datetime.date):
+            return o.isoformat()
+        elif isinstance(o, datetime.time):
+            if is_aware(o):
+                raise ValueError("JSON can't represent timezone-aware times.")
+            r = o.isoformat()
+            if o.microsecond:
+                r = r[:12]
+            return r
+        elif isinstance(o, datetime.timedelta):
+            return duration_iso_string(o)
+        elif isinstance(o, decimal.Decimal):
+            return str(o)
+        elif isinstance(o, uuid.UUID):
+            return str(o)
+        elif isinstance(o, Promise):
+            return six.text_type(o)
+        elif isinstance(o, CallableBool):
+            return bool(o)
+        else:
+            return super(DjangoJSONEncoder, self).default(o)
+
+
+class SignView(View):
+    """
+    """
+    def __init__(self):
+        self.client_id = "1088"
+        self.client_key = "3dea5896-0a5e-453f-821e-4db555c7cdb4"
+
+    def get_ip(self,request):
+        if request.META.has_key('HTTP_X_REAL_IP'):
+            ip = request.META['HTTP_X_REAL_IP']
+        elif request.META.has_key('HTTP_X_FORWARDED_FOR'):
+            ip = request.META['HTTP_X_FORWARDED_FOR']
+        else:
+            ip = request.META['REMOTE_ADDR']
+        return ip
+
+    #def gen_sign(self,timestamp):
+    #    """
+    #    """
+    #    signstr = "client_id={}client_key={}timestamp={}"\
+    #        .format(self.client_id,self.client_key,timestamp)
+    #    md5 = hashlib.md5(signstr).hexdigest().upper()
+    #    return md5
+
+    def gen_sign(self,params):
+        """
+        """
+        params.pop("sign")
+        keys = params.keys()
+        keys.sort()
+        signstr = ""
+        for key in keys:
+            signstr += u"{}={}&".format(key,params.get(key))
+        signstr += "key={}".format(self.client_key)
+        md5 = hashlib.md5(signstr).hexdigest().upper()
+        return md5
+
+    def check_auth(self,handler,request,*args,**kwargs):
+        """鉴权
+        """
+        params = request.json
+        sign = params.get("sign")
+
+        if self.gen_sign(params) != sign:
+            return JsonResponse({"status":"error","message":u"签名错误,sign的值错误!"})
+
+        return handler(request,*args,**kwargs)
+
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        """
+        """
+        if request.method.lower() in self.http_method_names:
+            handler = getattr(self, request.method.lower(), self.http_method_not_allowed)
+            body = request.body if hasattr(request, "body") else ""
+            if request.content_type == "application/x-www-form-urlencoded":
+                info = http.QueryDict(body)
+            elif request.content_type == "application/json":
+                info = json.loads(body) if body else {}
+            else:
+                info = {}
+
+            setattr(request, "json", info)
+        else:
+            handler = self.http_method_not_allowed
+        return self.check_auth(handler,request,*args,**kwargs)
+
+
+
+class AuthView(View):
+
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        """
+        @attention: as_view()方法使用该方法来分发不同http method,添加异常处理及登陆校验
+        """
+        if request.method.lower() in self.http_method_names:
+            handler = getattr(self, request.method.lower(), self.http_method_not_allowed)
+        else:
+            handler = self.http_method_not_allowed
+        if request.META.get("HTTP_TOKEN") == "7dpHIhpweckghdoSvrXwMftcjZRIzKwJ":
+            return api_wapper(handler, request, False, *args, **kwargs)
+        else:
+            return api_wapper(handler, request, True, *args, **kwargs)
+
+
+class BaseView(View):
+
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        """
+        @attention: as_view()方法使用该方法来分发不同http method,添加异常处理及登陆校验
+        """
+        if request.method.lower() in self.http_method_names:
+            handler = getattr(self, request.method.lower(), self.http_method_not_allowed)
+        else:
+            handler = self.http_method_not_allowed
+
+        return api_wapper(handler, request, False, *args, **kwargs)
+
+
+class UploadView(View):
+    
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        """
+        @attention: as_view()方法使用该方法来分发不同http method,添加异常处理及登陆校验
+        """
+        if request.method.lower() in self.http_method_names:
+            handler = getattr(self, request.method.lower(), self.http_method_not_allowed)
+        else:
+            handler = self.http_method_not_allowed
+
+        return upload_wapper(handler,request,True, *args, **kwargs)
+
+
+class InnerView(View):
+    
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        """
+        @attention: as_view()方法使用该方法来分发不同http method,添加异常处理及登陆校验
+        """
+        if request.method.lower() in self.http_method_names:
+            handler = getattr(self, request.method.lower(), self.http_method_not_allowed)
+            if request.META.get("HTTP_TOKEN") != "7dpHIhpweckghdoSvrXwMftcjZRIzKwJ":
+                handler = self.http_method_not_allowed
+        else:
+            handler = self.http_method_not_allowed
+
+        return api_wapper(handler, request, False, *args, **kwargs)
+
+
+def show_history(request):
+    logined_history = cache.get("logined_history", {})
+    for k, v in logined_history.iteritems():
+        logger.info("k: %s, v: %s", str(k), str(v))
+    logger.info("current session: %s", str(request.session.session_key))
+    ss = SessionStore(request.session.session_key)
+    for k, v in ss.iteritems():
+        logger.info("k: %s, v: %s", str(k), str(v))
+
+
+def api_wapper(handler, request, is_vauth, *args, **kwargs):
+    """
+    @attention: 调试API时使用的装饰器
+    """
+    req_path = request.META["PATH_INFO"]
+    ip = request.META.get("HTTP_X_REAL_IP","")
+    #if is_vauth and not request.user.is_authenticated() and not ip in ["10.10.19.98","10.10.19.97","10.10.19.84"] and not "/report/data" in req_path:
+    if is_vauth and not request.user.is_authenticated():
+        return HttpResponse(status=403)
+        return to_redirect("/")
+#     logger.info("request user: %s", str(request.user))
+#     if request.method == "GET":
+#         logger.info("request.GET: %s", str(request.GET))
+# 
+#     if request.method == "POST" or request.method == "PUT":
+#         logger.info("request.body: %s", str(request.body))
+
+    print "request.content_type: ", request.content_type
+
+    body = request.body if hasattr(request, "body") else ""
+    if "x-www-form-urlencoded" in request.content_type:
+        info = http.QueryDict(body).dict()
+        if not info:
+            info = request.GET.dict()
+    elif "application/json" in request.content_type:
+        info = json.loads(body) if body else {}
+        if not info:
+            info = request.GET.dict()
+    else:
+        try:
+            info = json.loads(body) if body else {}
+            if not info:
+                info = request.GET.dict()
+        except:
+            info = {}
+
+    setattr(request, "json", info)
+    setattr(request, "ip", get_ip(request))
+#     if request.method == "POST" or request.method == "PUT":
+#         logger.info("request.json: %s", str(request.json))
+
+    # import operation_log.op_log_exception as le
+    try:
+        ret = handler(request, *args, **kwargs)
+#         logger.info("ret: %s", str(ret.content))
+        return ret
+    except error_info.TipException as e:
+        return to_normal_fail(e.msg)
+    except ce.ReasonException as e:
+        # operation log here
+        user = request.user
+        name = user.realname
+        email = user.name
+        return to_normal_fail(e.msg)
+    except ce.SpecialReasonException as e:
+        user = request.user
+        name = user.realname
+        email = user.name
+        e.inner_info.update({'reason': e.msg})
+        return to_normal_fail(e.msg)
+    except:
+        logger.info("request.json: %s", str(request.json))
+        traceback.print_exc()
+        return to_normal_fail("system error!")
+
+
+def upload_wapper(handler,request,is_vauth, *args, **kwargs):
+    """
+    @attention: 上传使用
+    """
+    logger.info("request.GET: %s", str(request.GET))
+    if is_vauth and (not request.user.is_authenticated()):
+        return to_redirect("/")
+    info = {}
+    setattr(request, "json", info)
+    setattr(request, "ip", get_ip(request))
+    try:
+        ret = handler(request, *args, **kwargs)
+        logger.info("ret: %s", str(ret.content))
+        return ret
+    except error_info.TipException as e:
+        return to_normal_fail(e.msg)
+    except le.ReasonException as e:
+        # operation log here
+        user = request.user
+        name = user.realname
+        email = user.name
+        return to_normal_fail(e.msg)
+    except le.SpecialReasonException as e:
+        user = request.user
+        name = user.realname
+        email = user.name
+        e.inner_info.update({'reason': e.msg})
+        return to_normal_fail(e.msg)
+    except:
+        traceback.print_exc()
+        return to_normal_fail("system error!")
+
+
+def to_suc(data=""):
+    info = {}
+    info["data"] = data
+    info["status"] = "suc"
+    return JsonResponse(info,encoder=CusDjangoJSONEncoder)
+
+def to_suc_msg(data="",msg=""):
+    info = {}
+    info["data"] = data
+    info["msg"] = msg
+    info["status"] = "suc"
+    return JsonResponse(info)
+
+
+def to_normal_fail(msg=""):
+    info = {}
+    info["status"] = "error"
+    info["message"] = msg
+    return JsonResponse(info)
+
+def to_fail(e=None):
+    info = {}
+    info["status"] = "error"
+    if isinstance(e,ce.TipException):
+        info["message"] = e.msg
+    else:
+        info["message"] = str(e)
+    return JsonResponse(info)
+
+def to_redirect(curl):
+    info = {}
+    info["status"] = "red"
+    info["url"] = curl
+    return JsonResponse(info)
+
+
+def raw_json(data):
+    return JsonResponse(data, safe=False)
+
+def tracefail():
+    traceback.print_exc()
+
+EXCEL_TYPE = "application/vnd.ms-excel"
+
+
+def stream_file(content, content_type, file_name):
+    """
+    输出文件
+    :param content: 内容 StringIO 类型
+    :param content_type: 类型  eg: "application/vnd.ms-excel"
+    :param file_name: 文件名（需指定后缀）
+    """
+    response = HttpResponse(content=content, content_type=content_type)
+    response['Content-Disposition'] = 'attachment; filename={}'.format(file_name)
+    return response
+
+
+def check_permission(*codes):
+    code_set = set(codes)
+
+    def function_wapper(fn):
+        def wapper(cls, request, *args, **kargs):
+            try:
+                user = UserInfo.objects.filter(pk=request.user.id).first()
+                plist = json.loads(user.role.permission_codes) if user else []
+            except Exception as e:
+                plist = []
+            pset = set(plist)
+            #logger.info("have permission: %s, need permission: %s", str(plist), str(code_set))
+            flag = True
+            for code in code_set:
+                res = filter(lambda item: re.match(item, code), pset)
+                if not (code in code_set):
+                    flag = False
+                    break
+            if flag:
+                return fn(cls, request, *args, **kargs)
+            else:
+                return to_normal_fail(u"该用户没有此操作权限")
+
+        return wapper
+
+    return function_wapper
+
+
+def get_ip(request):
+    if request.META.has_key('HTTP_X_REAL_IP'):
+        ip = request.META['HTTP_X_REAL_IP']
+    elif request.META.has_key('HTTP_X_FORWARDED_FOR'):
+        ip = request.META['HTTP_X_FORWARDED_FOR']
+    else:
+        ip = request.META['REMOTE_ADDR']
+    return ip

src/common/custom_tools.py

@@ -0,0 +1,227 @@
+#coding=utf-8
+'''
+Created on 2017年11月14日
+
+@author: bailiangjun
+'''
+def change_dict_p2m(item,p2m,ms=[],ems=[]):
+    """
+    @attention: 把指定信息的关键字换成对应的模型字段m
+    @param item: 默认为 {p:值}
+    @note: 
+    {p:值} ==> {m:值}
+    如果传入指定序列ps,则只取序列中的值
+    如果传入指定序列eps,则取除eps以外的值
+    """
+    if not item:
+        return {}
+    
+    if not ms:
+        ms = set(p2m.itervalues())
+    temp = {p:m for p,m in p2m.iteritems() if (m in ms) and (m not in ems)}
+    return {m:item.get(p,"") for p,m in temp.iteritems()}
+
+def change_dict_m2p(item,p2m,ms=[],ems=[]):
+    """
+    @attention: 把指定信息的关键字换成对应的api协议字段
+    @param item: 默认为 {m:值}
+    @note: 
+    {m:值} ==> {p:值}
+    如果传入指定序列ps,则只取序列中的值
+    如果传入指定序列eps,则取除eps以外的值
+    """
+    if not item:
+        return {}
+    
+    if not ms:
+        ms = set(p2m.itervalues())
+    
+    temp = {p:m for p,m in p2m.iteritems() if (m in ms) and (m not in ems)}
+    return {p:item.get(m,"") for p,m in temp.iteritems()}
+
+
+def change_list_p2m(item,p2m,ms=[],ems=[]):
+    """
+    @attention: 把指定信息的关键字换成对应的模型字段m,针对列表
+    @param item: 默认为 {p:值}
+    @note: 
+    {p:值} ==> {m:值}
+    如果传入指定序列ps,则只取序列中的值
+    如果传入指定序列eps,则取除eps以外的值
+    """
+    if not item:
+        return []
+    
+    if not ms:
+        ms = set(p2m.itervalues())
+    
+    temp = {p:m for p,m in p2m.iteritems() if (m in ms) and (m not in ems)}
+
+    return map(lambda x:{m:x.get(p,"") for p,m in temp.iteritems()}, item)
+
+
+def change_list_m2p(item,p2m,ms=[],ems=[]):
+    """
+    @attention: 把指定信息的关键字换成对应的api协议字段,针对列表
+    @param item: 默认为 {m:值}
+    @note: 
+    {m:值} ==> {p:值}
+    如果传入指定序列ps,则只取序列中的值
+    如果传入指定序列eps,则取除eps以外的值
+    """
+    if not item:
+        return []
+    
+    if not ms:
+        ms = set(p2m.itervalues())
+    
+    temp = {p:m for p,m in p2m.iteritems() if (m in ms) and (m not in ems)}
+    return map(lambda x:{p:x.get(m,"") for p,m in temp.iteritems()}, item)
+
+
+def change_node_m2p(item, p2m, ms=[], ems=[]):
+    """
+    @attention: 把指定信息的关键字换成对应的api协议字段,针对列表
+    @param item: 默认为 {m:值}
+    @note:
+    {m:值} ==> {p:值}
+    如果传入指定序列ps,则只取序列中的值
+    如果传入指定序列eps,则取除eps以外的值
+    """
+    if not item:
+        return []
+
+    if not ms:
+        ms = set(p2m.itervalues())
+
+    temp = {p: m for p, m in p2m.iteritems() if (m in ms) and (m not in ems)}
+    return map(lambda x: {p: x.get(m, "") for p, m in temp.iteritems()}, item)
+
+
+def transfer_top_info(info, lkey, tkey):
+    """
+    @attention: 转换top数据给前端
+    @note:
+            由 二维数组[[标签,值],[]]---> [{label:标签,"times":值}]
+    """
+    return [{"name": item[lkey], "value": item[tkey]} for item in info]
+
+
+def transfer_list_info(info, params):
+    """
+    @attention: 转换趋势数据给前端
+    """
+    if not info:
+        return {"axis_x": [], "axis_y": {}}
+
+    if isinstance(params, list):
+        params = {item: [item] for item in params}
+
+    print params
+
+    pkeys = params.keys()
+    if len(pkeys) == 1:
+        key = pkeys[0]
+        inkeys = params[key]
+        inkeys = inkeys if inkeys else [key]
+        v = info[key]
+        axis_x = [item["dt"] for item in v]
+        axis_y = {}
+        for ik in inkeys:
+            axis_y[ik] = [item[ik] for item in v]
+    else:
+        axis_y = {}
+        for key in pkeys:
+            inkeys = params[key]
+            inkeys = inkeys if inkeys else [key]
+            v = info[key]
+            axis_x = [item["dt"] for item in v]
+            for ik in inkeys:
+                axis_y[ik] = [item[ik] for item in v]
+
+    return {"axis_x": axis_x, "axis_y": axis_y}
+
+
+def classify_access_statistic_info(info):
+    """
+    @attention: 区分access统计数据项
+    """
+    total_values = []
+    list_values = []
+    top_values = {}
+    if info.has_key("total_hit_flow"):
+        total_values.append("hit_flow")
+
+    if info.has_key("total_hit_num"):
+        total_values.append("hit_total")
+
+    if info.has_key("total_req_flow"):
+        total_values.append("req_flow")
+
+    if info.has_key("total_req_num"):
+        total_values.append("req_total")
+
+    if info.has_key("tendency_pv"):
+        list_values.append("pageview")
+
+    if info.has_key("tendency_quote"):
+        list_values.append("search")
+
+    if info.has_key("tendency_req_flow"):
+        list_values.append("req_flow")
+        list_values.append("hit_flow")
+
+    if info.has_key("tendency_req_num"):
+        list_values.append("req_total")
+        list_values.append("hit_total")
+
+    if info.has_key("tendency_ip"):
+        list_values.append("ip_num")
+
+    if info.get("top_location"):
+        top_values["location"] = int(info["top_location"])
+    if info.get("top_location_flow"):
+        top_values["location"] = int(info["top_location_flow"])
+
+    return total_values, list_values, top_values
+
+
+def classify_waf_statistic_info(info):
+    """
+    @attention: 区分waf统计数据项
+    """
+    total_values = []
+    list_values = []
+    top_values = {}
+    if info.has_key("level"):
+        total_values.append("level")
+
+    if info.has_key("total_cc") or info.has_key("total_web"):
+        total_values.append("att_num")
+
+    if info.has_key("total_ip"):
+        total_values.append("ip_num")
+
+    if info.has_key("tendency_attack"):
+        list_values.append("att_num")
+
+    if info.get("top_ip"):
+        top_values["src_ip"] = int(info["top_ip"])
+
+    if info.get("top_location"):
+        top_values["location"] = int(info["top_location"])
+
+    if info.get("top_type"):
+        top_values["rule"] = int(info["top_type"])
+
+    if info.get("top_site"):
+        top_values["domain"] = int(info["top_site"])
+
+    return total_values, list_values, top_values
+
+
+def dict_unicode2str(**params):
+    item = {}
+    for k in params:
+        item[str(k)] = str(params[k])
+    return str(item)

src/common/error_info.py

@@ -0,0 +1,40 @@
+#coding=utf-8
+'''
+@attention: 异常定义模块
+'''
+class TipException(Exception):
+    """
+    @attention: 提示类异常,用于业务阻断时,直接返回请求
+    """
+    def __init__(self,msg):
+        self.msg = msg
+        
+    def show_msg(self):
+        return self.msg
+
+class ReasonException(Exception):
+    """
+    只需要原因的日志
+    """
+
+    def __init__(self, msg, module_name, msg_template):
+        """
+
+        :param msg:
+        :param module_name:
+        :param msg_template:  必须接收 reason参数
+        """
+        self.msg = msg
+        self.module_name = module_name
+        self.msg_template = msg_template
+
+
+class SpecialReasonException(Exception):
+    """
+    特殊异常
+    """
+    def __init__(self, msg, module_name, msg_template, inner_info):
+        self.msg_template = msg_template
+        self.module_name = module_name
+        self.msg = msg
+        self.inner_info = inner_info

src/common/excel_output.py

@@ -0,0 +1,33 @@
+# coding:utf-8
+import StringIO
+import xlwt
+import zipfile
+import datetime
+import random, string
+import os
+from ftplib import FTP
+
+def save2memory(en_cn_map, content):
+    """
+    保存内容到xls里面
+    :param en_cn_map:  英文中文对应字典 {"name": "名字"}, OrderedDict
+    :param content:
+    :return:
+    """
+    wb = xlwt.Workbook(encoding="utf-8")
+    sheet = wb.add_sheet('info')
+    sort_head = en_cn_map.keys()
+    for i, item in enumerate(en_cn_map):
+        sheet.write(0, i, en_cn_map.get(item))
+    for row, item in enumerate(content):
+        sort_key = item.keys()
+        for key in sort_key:
+            if key not in sort_head:
+                continue
+            sheet.write(row+1, sort_head.index(key), item.get(key, ''))
+    output = StringIO.StringIO()
+    wb.save(output)
+    return output.getvalue()
+
+if __name__ == '__main__':
+    pass

src/common/logger.py

@@ -0,0 +1,39 @@
+#coding=utf-8
+'''
+Created on 2017年10月24日
+
+@author: bailiangjun
+'''
+import logging
+
+import settings
+
+logger = logging.getLogger('cloudwaf_defense')
+
+# 设置logger的level为DEBUG
+if settings.DEBUG:
+    logger.setLevel(logging.DEBUG)
+else:
+    logger.setLevel(logging.WARN)
+
+# 创建一个输出日志到控制台的StreamHandler
+hdr = logging.StreamHandler()
+
+# str_info = ["name","levelno","levelname","pathname","filename","module","funcName","created","asctime","threadName","message"]
+# int_info = ["lineno","msecs","relativeCreated","thread","process"]
+
+formatter = logging.Formatter("[%(asctime)s %(levelname)s]<%(name)s %(pathname)s %(lineno)s>:%(message)s")
+hdr.setFormatter(formatter)
+# 给logger添加上handler
+logger.addHandler(hdr)
+    
+def critical(msg,*args):
+    """
+    @attention: 考虑添加邮件系统/短信系统通知
+    """
+    logger.critical(msg,*args)
+
+
+
+if __name__ == '__main__':
+    logger.critical("ss%s")

src/common/models.py

@@ -0,0 +1,180 @@
+# coding=utf-8
+'''
+'''
+import utils.constant as uconst
+from django.db import models
+
+
+# ---------------- 用户部分 ------------------
+class UserInfo(models.Model):
+    # 基础属性
+    name = models.CharField(u"名字", max_length=255, blank=True)
+    realname = models.CharField(u"真实名称", max_length=255, blank=True, null=True)
+    email = models.CharField(u"邮箱", max_length=255, blank=True)
+    phone = models.CharField(u"手机号", max_length=64, blank=True)
+    parent_id = models.CharField(u"创建人ID", max_length=16, blank=True)
+    remark = models.TextField(u"备注", blank=True)
+
+    last_login = models.CharField(u"上次登录时间", max_length=128, blank=True, db_column='last_login_time')
+    last_login_ip = models.CharField(u"上次登录IP", max_length=128, blank=True)
+    is_active = models.CharField(u"是否激活可用", max_length=4, default="1")  # 0-未激活;1-已激活
+    # 功能权限及数据权限
+    role = models.ForeignKey("Role", verbose_name=u"所属角色", blank=True, null=True)
+
+    cperson = models.CharField(u"创建人", max_length=255, blank=True)
+    ctime = models.DateTimeField(u"创建时间", auto_now_add=True)
+
+    class Meta:
+        db_table = "adminuser"
+        verbose_name = u"用户信息"
+        app_label = "common"
+
+    def __str__(self):
+        return u"{}){}".format(self.id, self.email)
+
+    def is_authenticated(self):
+        return True
+
+    def is_anonymous(self):
+        return False
+
+
+class Category(models.Model):
+    name = models.CharField(u"组织名称",max_length=256,blank=True)
+    pid = models.CharField(u"父节点ID",max_length=16,blank=True)
+    isshow = models.BooleanField(u"是否显示",blank=True,null=False)
+    isnav = models.BooleanField(u"是否导航",blank=True,null=False)
+    order = models.IntegerField(u"排序字段",blank=True,null=True,default=0)
+
+    cid = models.IntegerField(u"创建人ID",blank=True)
+    cperson = models.CharField(u"创建人",max_length=255,blank=True)
+    ctime = models.DateTimeField(u"创建时间",auto_now_add=True)
+
+    class Meta:
+        db_table = "category"
+        verbose_name = u"组织信息"
+        app_label = "common"
+
+
+class Permission(models.Model):
+    """权限
+    """
+    name = models.CharField(u"权限名称", max_length=50, blank=True)
+    codename = models.CharField(u"权限代码", max_length=255, blank=True)
+    remark = models.CharField(u"备注", max_length=255, blank=True, null=True, default="")
+    order = models.IntegerField(u"排序字段", default=1, blank=True, null=True)
+    pid = models.CharField(u"父节点ID",max_length=16,blank=True,null=True)
+
+    class Meta:
+        db_table = "permission"
+        app_label = "common"
+
+
+class Role(models.Model):
+    permission = models.ManyToManyField(Permission, related_name="permission_role")
+    name = models.CharField(u"角色名称", max_length=255, blank=True)
+    desc = models.TextField(u"角色描述", blank=True)
+
+    cid = models.IntegerField(u"创建人ID",blank=True)
+    cperson = models.CharField(u"创建人", max_length=255, blank=True)
+    ctime = models.DateTimeField(u"创建时间", auto_now_add=True)
+
+    class Meta:
+        db_table = "role"
+        verbose_name = u"角色信息"
+        app_label = "common"
+
+
+class OperationLogConfig(models.Model):
+    """操作日志模板配置
+    """
+    op_view = models.CharField(u"操作模块", max_length=100)
+    op_action_flag = models.CharField(u"操作类型", max_length=50)
+    op_module = models.CharField(u"操作模块",max_length=255,blank=True,null=True)
+    op_template = models.TextField(u"操作日志模板")
+    op_url = models.CharField(u"URL", max_length=255)
+    op_response = models.TextField(u"URL", max_length=255, blank=True, null=True)
+    order = models.IntegerField(u"排序字段",blank=True,null=True)
+
+    ctime = models.DateTimeField(u"创建时间", auto_now_add=True)
+
+    class Meta:
+        db_table = "oplog_config"
+        verbose_name = u"操作日志配置"
+
+
+class OperationLog(models.Model):
+    """操作日志
+    """
+    op_user_name = models.CharField(u"操作员名称",max_length=255)
+    op_user_realname = models.CharField(u"操作员姓名",max_length=255)
+    op_user_id = models.IntegerField(u"操作员id")
+    op_user_ip = models.CharField(u"操作员ip",max_length=100)
+    op_config = models.ForeignKey(OperationLogConfig)
+    op_request = models.TextField(u"request body", default="")
+    is_suc = models.BooleanField(u"是否成功", default=True)
+    error_msg = models.TextField(u"错误信息", max_length=255, blank=True, null=True)
+    ctime = models.DateTimeField(u"创建时间", auto_now_add=True)
+
+    class Meta:
+        db_table = "oplog"
+        verbose_name = u"操作日志"
+
+
+class EmailNotice(models.Model):
+    """邮件通知
+    """
+    receiver = models.TextField(u"接收人集合", blank=True)
+    receiver_name = models.CharField(u'接收人id', max_length=256, blank=True)
+    subject = models.CharField(u'邮件主题', blank=True, max_length=128)
+    content = models.TextField(u"邮件内容", max_length=32, blank=True)
+    is_suc = models.BooleanField(u"是否成功", default=False)
+    error_msg = models.TextField(u'报错信息', blank=True,null=True)
+    send_type = models.CharField(u"发送方式（手动、自动）", max_length=12, blank=True)
+
+    cid = models.IntegerField(u"创建人ID",blank=True,null=True)
+    cperson = models.CharField(u'邮件主题', blank=True, max_length=128,null=True)
+    ctime = models.DateTimeField(u"创建时间(发送时间)", auto_now_add=True)
+
+    class Meta:
+        db_table = "email_notice"
+        verbose_name = u"邮件发送记录"
+
+class MsgNotice(models.Model):
+    """短信通知
+    """
+    receiver = models.TextField(u"接收人集合", blank=True)
+    receiver_name = models.CharField(u'接收人id', max_length=256, blank=True)
+    template = models.CharField(u'模板id', blank=True, max_length=128)
+    content = models.TextField(u"邮件内容", max_length=32, blank=True)
+    is_suc = models.BooleanField(u"是否成功", default=False)
+    error_msg = models.TextField(u'报错信息', blank=True,null=True)
+    send_type = models.CharField(u"发送方式（手动、自动）", max_length=12, blank=True)
+
+    cid = models.IntegerField(u"创建人ID",blank=True,null=True)
+    cperson = models.CharField(u'邮件主题', blank=True, max_length=128,null=True)
+    ctime = models.DateTimeField(u"创建时间(发送时间)", auto_now_add=True)
+
+    class Meta:
+        db_table = "msg_notice"
+        verbose_name = u"短信通知"
+
+
+class SysNotice(models.Model):
+    """系统通知
+    """
+    title = models.CharField(u'标题', max_length=256, blank=True)
+    content = models.TextField(u"内容", max_length=32, blank=True)
+    viewer = models.TextField(u"查看人", max_length=32, blank=True,null=True)
+    to = models.TextField(u"接收人", max_length=32, blank=True,null=True)
+
+    cid = models.IntegerField(u"创建人ID",blank=True,null=True)
+    cperson = models.CharField(u'邮件主题', blank=True, max_length=128,null=True)
+    ctime = models.DateTimeField(u"创建时间(发送时间)", auto_now_add=True)
+
+    class Meta:
+        db_table = "sys_notice"
+        verbose_name = u"系统通知"
+
+
+

src/common/views.py

@@ -0,0 +1,35 @@
+# coding=utf-8
+from __future__ import unicode_literals
+
+import json
+import uuid
+import random
+
+from django.contrib import auth
+from django.core.cache import cache
+
+from common import core_views as cv
+from common.models import Industry 
+
+class IndustryView(cv.BaseView):
+    def get(self, request):
+        """获取行业数据接口
+        """
+        qdata = request.json
+        idset = Industry.objects.filter(status="normal").values("id","name")
+        rdata = list(idset)
+        return cv.to_suc(rdata)
+
+
+class WsNotifyView(cv.BaseView):
+    def post(self,request):
+        """通知前端ws即时信息
+        rating:安全级别
+        statistic:分析相关
+        attack:攻击相关
+        """
+        qdata = request.json
+        type = qdata.get("type","attack")
+        return cv.to_suc({})
+
+

src/common/ws_notice.py

@@ -0,0 +1,9 @@
+#coding:utf-8
+
+import json
+from channels import Group
+
+def send_ws_msg(channel,msg=""):
+    """
+    """
+    Group(channel).send({'text': json.dumps({'type': msg})})

src/dashboard/test_case/test_control_dashboard.py

@@ -0,0 +1,52 @@
+# -*-coding:utf-8-*-
+
+import django
+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+django.setup()
+
+import dashboard.control_dashboard as ctd
+
+
+def test_get_website_statisic():
+    ret = ctd.get_nssa_website_statistics(18)
+    print ret
+
+
+def test_get_website_available():
+    ret = ctd.get_nssa_website_available(18)
+    print ret
+
+
+def test_get_website_risk():
+    ret = ctd.get_nssa_website_risk(18)
+    print ret
+
+
+def test_get_website_rist_latest():
+    ret = ctd.get_nssa_website_risk_latest(18)
+    print ret
+
+
+def test_get_website_bug():
+    ret = ctd.get_nssa_website_bug(18)
+    print ret
+
+
+def test_get_nssa_website_tendency():
+    ret = ctd.get_nssa_website_tendency(18)
+    print ret
+
+
+if __name__ == '__main__':
+    print "=========================开始============================"
+
+    # test_get_website_statisic()
+    # test_get_website_available()
+    # test_get_website_risk()
+    # test_get_website_rist_latest()
+    # test_get_website_bug()
+    test_get_nssa_website_tendency()
+
+    print "=========================结束============================"

src/dashboard/urls_backstage.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+'''
+'''
+from django.conf.urls import url
+from dashboard import views_backstage
+
+urlpatterns = [
+    # # 首页或其他
+    # url(r'^info$', views_backstage.WebsiteView.as_view()),
+
+
+    #态势接口
+    url(r'^nssa/website$', views_backstage.WebsiteStatisticsView.as_view()),
+    url(r'^nssa/available$', views_backstage.WebsiteAvailableAlertView.as_view()),
+    url(r'^nssa/risk$', views_backstage.WebsiteRiskView.as_view()),
+    url(r'^nssa/risk/latest$', views_backstage.WebsiteRiskLatestView.as_view()),
+    url(r'^nssa/bug$', views_backstage.WebsiteBugView.as_view()),
+    url(r'^nssa/tendency$', views_backstage.WebsiteTendencyView.as_view()),
+]

src/dashboard/views_backstage.py

@@ -0,0 +1,97 @@
+# -*-coding:utf-8-*-
+
+
+import common.common_functions as ccf
+import common.error_info as ce
+from common import core_views as cv
+from dashboard import control_dashboard as ctd
+
+
+class WebsiteStatisticsView(cv.AuthView):
+    def get(self, request):
+        """
+        #态势网站监测基本统计
+        参数：无
+        """
+        user = request.user
+        uid = user.id
+        try:
+            ret = ctd.get_nssa_website_statistics(uid)
+            return cv.to_suc(ret)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WebsiteAvailableAlertView(cv.AuthView):
+    def get(self, request):
+        """
+        #态势网站可用性告警
+        参数：无
+        """
+        user = request.user
+        uid = user.id
+        try:
+            ret = ctd.get_nssa_website_available(uid)
+            return cv.to_suc(ret)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WebsiteRiskView(cv.AuthView):
+    def get(self, request):
+        """
+        #态势风险网站TOP10
+        参数：无
+        """
+        user = request.user
+        uid = user.id
+        try:
+            ret = ctd.get_nssa_website_risk(uid)
+            return cv.to_suc(ret)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WebsiteRiskLatestView(cv.AuthView):
+    def get(self, request):
+        """
+        #态势最新威胁信息
+        参数：无
+        """
+        user = request.user
+        uid = user.id
+        try:
+            ret = ctd.get_nssa_website_risk(uid)
+            return cv.to_suc(ret)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WebsiteBugView(cv.AuthView):
+    def get(self, request):
+        """
+        #态势网站漏洞信息
+        参数：无
+        """
+        user = request.user
+        uid = user.id
+        try:
+            ret = ctd.get_nssa_website_bug(uid)
+            return cv.to_suc(ret)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WebsiteTendencyView(cv.AuthView):
+    def get(self, request):
+        """
+        #态势风险趋势
+        参数：无
+        """
+        user = request.user
+        uid = user.id
+        try:
+            ret = ctd.get_nssa_website_tendency(uid)
+            return cv.to_suc(ret)
+        except Exception as e:
+            return cv.to_fail(e)

src/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

src/operation_log/OpLogMiddleware.py

@@ -0,0 +1,81 @@
+#-*-coding:utf-8 -*-
+"""
+"""
+import re,json
+import common.models as cm
+import common.core_views as cv
+
+class OpLogMiddleware(object):
+    def __init__(self,get_response):
+        '''
+        '''
+        self.get_response = get_response
+
+    def __call__(self,request):
+        '''
+        '''
+        response = self.get_response(request)
+        path = request.path.strip()
+        if path not in ["/api/doc/","/favicon.ico","/api/hook/"] and "/api/report/download" not in path:
+            self.record_oplog(request,response)
+        return response
+
+    def record_oplog(self,request,response):
+        """
+        """
+        op_view = re.findall(r'func=([\.\w]+)\b',str(request.resolver_match))[0]
+        op_action_flag = request.method
+        op_url = request.path.strip()
+        #op_url = op_url.split("?")[0]
+        if request.META.get("CONTENT_TYPE","") == "application/json":
+            reqdata = request.body if request.body else {}
+        else:
+            if op_action_flag == "POST":
+                reqdata = request.POST.dict()
+            elif op_action_flag == "GET":
+                reqdata = request.GET.dict()
+            else:
+                reqdata = request.body if request.body else {}
+        reqdata = json.dumps(reqdata)
+        try:
+            op_user_name = request.user.name if request.user and request.user.id else reqdata.get("name",u"匿名")
+            op_user_realname = request.user.realname if request.user and request.user.id else reqdata.get("realname",u"匿名")
+        except:
+            op_user_name = ""
+            op_user_realname = ""
+        op_user_id = request.user.id if request.user else 0 
+        op_user_id = op_user_id if op_user_id else 0
+        op_user_ip = cv.get_ip(request)
+        op_request = json.dumps(reqdata)
+        try:
+            response_data = json.loads(response.getvalue())
+        except:
+            response_data = {}
+        if response_data.get("status") == "suc":
+            is_suc = 1
+            error_msg = ""
+        else:
+            is_suc = 0
+            error_msg = response_data.get("message")
+        #
+        last = cm.OperationLogConfig.objects.order_by("order").last()
+        last_order = last.order if last else 0
+        obj,flag = cm.OperationLogConfig.objects\
+                .get_or_create(**{"op_view":op_view,"op_action_flag":op_action_flag,"op_url":op_url})
+        if flag:
+            obj.order = last_order
+        if not obj.op_response:
+            if response_data.get("status") == "suc":
+                obj.op_response = json.dumps(response_data)
+                obj.save()
+        cm.OperationLog.objects.create(
+            op_user_name = op_user_name,
+            op_user_realname = op_user_realname,
+            op_user_id = op_user_id,
+            op_user_ip = op_user_ip,
+            op_request = op_request,
+            op_config = obj, 
+            is_suc = is_suc,
+            error_msg = error_msg
+        )
+

src/operation_log/control_oplog.py

@@ -0,0 +1,45 @@
+# -*-coding:utf-8-*-
+import datetime
+
+import common.error_info as ce
+import common.models as cm
+import common.common_functions as ccf
+import common.common_control as ccc
+
+
+def get_oplog_list(request):
+    filter_info = request.json
+    qset = cm.OperationLog.objects.all()
+    if filter_info.get("op_user_name"):
+        qset = qset.filter(op_user_name__icontains=filter_info.get("op_user_name"))
+    if filter_info.get("op_user_ip"):
+        qset = qset.filter(op_user_ip=filter_info.get("op_user_ip"))
+    if filter_info.get("op_content"):
+        qset = qset.filter(op_content=filter_info.get("op_content"))
+    # if filter_info.get("stime"):
+    #     qset = qset.filter(ctime__gte=filter_info.get("stime"))
+    # if filter_info.get("etime"):
+    #     qset = qset.filter(ctime__lte=filter_info.get("etime"))
+    if filter_info.get("ctime", ""):
+        import web_site.tools as wst
+        query = wst.datetime_filter("ctime", {}, filter_info["ctime"])
+        qset = qset.filter(**query)
+    count = qset.count()
+    page = filter_info.get("page")
+    page_size = filter_info.get("page_size")
+    if page and page_size:
+        count,qset = ccc.get_page_qset(qset,page,page_size)
+    qdata = list(qset.values("op_user_name","op_user_realname","op_user_ip","ctime","is_suc"))
+    for qd in qdata:
+        qd["op_module"] = "系统管理"
+        qd["op_content"] = "删除root用户成功"
+
+    return count,qdata
+
+
+def get_warning_notice(*ids):
+    """
+    """
+    qset = cm.WarningNotice.objects.filter(id__in=ids)
+    qdata = qset.values()
+    return qdata

src/operation_log/urls_backstage.py

@@ -0,0 +1,11 @@
+# coding=utf-8
+'''
+'''
+from django.conf.urls import url
+from operation_log import views_oplog
+
+urlpatterns = [
+    #任务
+    url(r'^list$', views_oplog.OPlogListView.as_view()),
+]
+

src/operation_log/views_oplog.py

@@ -0,0 +1,30 @@
+#-*-coding:utf-8-*-
+import json
+import uuid
+from django.contrib import auth
+from django.core.cache import cache
+from django.conf import settings
+
+from common import core_views as cv
+from common.models import UserInfo
+import common.common_functions as ccf
+import common.error_info as ce
+import operation_log.control_oplog as cop
+
+
+class OPlogListView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取操作日志列表
+        @op_user_name:"名称",
+        @op_user_ip:"称",ip
+        @op_content:"称",操作详情
+        @ctime:"2020-03-04 00:00:00,2020-03-05 00:00:00" 时间范围
+        """
+        try:
+            count,res = cop.get_oplog_list(request)
+            return cv.to_suc({"total":count,"list":res})
+        except Exception as e:
+            return cv.to_fail(e)
+
+

src/script/start_collect_mtask_result.py

@@ -0,0 +1,36 @@
+#-*-coding:utf-8 -*-
+import time
+import os,sys
+import django
+
+sys.path.append(os.path.join("..",os.path.dirname(__file__)))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+django.setup()
+
+from start_collect_mtask_result_available import collect_results_available
+from start_collect_mtask_result_bug import collect_results_bug
+from start_collect_mtask_result_content import collect_results_content
+from start_collect_mtask_result_webshell import collect_results_webshell
+from start_collect_mtask_result_poc import collect_results_poc
+
+
+
+def main():
+    """
+    """
+    while True:
+        #可用性
+        collect_results_available()
+        #漏洞检测
+        collect_results_bug()
+        #事件检测
+        collect_results_content()
+        #webshell
+        collect_results_webshell()
+        #专项检测
+        collect_results_poc()
+        time.sleep(60)
+
+
+if __name__ == "__main__":
+    main()

src/script/start_collect_mtask_result_available.py

@@ -0,0 +1,78 @@
+#-*-coding:utf-8 -*-
+import json
+import os,sys
+import django
+
+sys.path.append(os.path.join("..",os.path.dirname(__file__)))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+django.setup()
+
+import common.models as cm
+import webray_service.control_webray as cwr
+
+
+def get_available_tasks():
+    """
+    """
+    wrtask = cm.WebRayTasks.objects.filter(type=1)
+    return wrtask
+
+
+def collect_results_available():
+    tasks = get_available_tasks()
+    for task in tasks:
+        siteid = task.siteid
+        task_id = task.id
+        website_id = task.website_id
+        wajob_res = cwr.get_wajob(siteid)
+        wa_rst_http_list = wajob_res.get("wa_rst_http_list")
+        wa_rst_http_list = sorted(wa_rst_http_list,key=lambda x:x["received_at"],reverse=True)
+        for rst in wa_rst_http_list:
+            taskid = rst.get("taskid")
+            jobid = rst.get("jobid")
+            logtype = rst.get("logtype")
+            httptime = rst.get("httptime")
+            received_at = rst.get("received_at")
+            isalert = rst.get("isalert")
+            trobj,flag = cm.AvailableTasksResult.objects.get_or_create(
+                    task_id = task_id, 
+                    taskid = taskid,
+                    siteid = siteid,
+                    jobid = jobid,
+                    isalert = isalert,
+                    website_id = website_id
+                )
+            trobj.logtype = logtype
+            trobj.httptime = httptime
+            trobj.received_at = received_at
+            trobj.save()
+            if trobj.isalert:
+                #生成可用性告警事件
+                cm.WarningEvent.objects.create(**{"event_type":"avaible",
+                        "task_type":1,
+                        "task_id":task_id,
+                        "taskid":taskid,
+                        "jobid":jobid,
+                        "taskname":task.name,
+                        "domain":task.site.split("//")[1].split(":")[0],
+                        "label":task.website_name,
+                        "organization":cm.WebSite.objects.filter(id=website_id).first().organization_name,
+                        "result":json.dumps([{"isalert":1,"httptime":httptime}])
+                        })
+        
+def main():
+    collect_results_available()
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+
+
+
+
+
+

src/script/start_collect_mtask_result_bug.py

@@ -0,0 +1,93 @@
+#-*-coding:utf-8 -*-
+import json
+import os,sys
+import django
+
+sys.path.append(os.path.join("..",os.path.dirname(__file__)))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+django.setup()
+
+import common.models as cm
+import webray_service.control_webray as cwr
+
+
+def get_available_tasks():
+    """
+    """
+    wrtask = cm.WebRayTasks.objects.filter(type=2)
+    return wrtask
+
+
+def collect_results_bug():
+    tasks = get_available_tasks()
+    for task in tasks:
+        print task,11111111111
+        siteid = task.siteid
+        task_id = task.id
+        taskid = task.taskid
+        website_id = task.website_id
+        print taskid,siteid,3333333333333
+        try:
+            wvjob_res = cwr.get_wvjob(siteid,taskid)
+            print wvjob_res,2222
+            if isinstance(wvjob_res,type({})):
+                jobreport = wvjob_res.get("jobreport")
+                vuln_list = wvjob_res.get("vuln_list")
+                taskid = jobreport.get("taskid")
+                jobid = jobreport.get("jobid")
+                logtype = jobreport.get("logtype")
+                received_at = jobreport.get("received_at")
+                starttime = jobreport.get("starttime")
+                endtime = jobreport.get("endtime")
+                total = jobreport.get("total")
+                high = jobreport.get("high")
+                mid = jobreport.get("mid")
+                low = jobreport.get("low")
+                info = jobreport.get("info")
+                isalert = jobreport.get("isalert")
+
+                trobj,flag = cm.BugTasksResult.objects.get_or_create(
+                        task_id = task_id, 
+                        taskid = taskid,
+                        siteid = siteid,
+                        jobid = jobid,
+                        website_id = website_id
+                    )
+                trobj.logtype = logtype
+                trobj.received_at = received_at
+                trobj.starttime = starttime
+                trobj.endtime = endtime
+                trobj.total = total
+                trobj.high = high
+                trobj.mid = mid
+                trobj.low = low
+                trobj.info = info
+                trobj.isalert = isalert
+                trobj.vuln_list = json.dumps(vuln_list)
+                trobj.save()
+                #生成漏洞告警事件
+                if trobj.vuln_list:
+                    cm.WarningEvent.objects.create(**{"event_type":"bug",
+                            "task_type":1,
+                            "task_id":task_id,
+                            "taskid":taskid,
+                            "jobid":jobid,
+                            "taskname":task.name,
+                            "domain":task.site.split("//")[1].split(":")[0],
+                            "label":task.website_name,
+                            "organization":cm.WebSite.objects.filter(id=website_id).first().organization_name,
+                            "result":trobj.vuln_list
+                            })
+
+        except Exception as e:
+            continue
+        
+
+def main():
+    """
+    """
+    collect_results()
+
+
+if __name__ == "__main__":
+    main()

src/script/start_collect_mtask_result_content.py

@@ -0,0 +1,76 @@
+#-*-coding:utf-8 -*-
+import json
+import os,sys
+import django
+
+sys.path.append(os.path.join("..",os.path.dirname(__file__)))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+django.setup()
+
+import common.models as cm
+import webray_service.control_webray as cwr
+
+
+def get_available_tasks():
+    """
+    """
+    wrtask = cm.WebRayTasks.objects.filter(type=3)
+    return wrtask
+
+
+def collect_results_content():
+    tasks = get_available_tasks()
+    for task in tasks:
+        siteid = task.siteid
+        task_id = task.id
+        taskid = task.taskid
+        website_id = task.website_id
+        try:
+            wcjob_res = cwr.get_wcjob(siteid,taskid)
+            print wcjob_res
+            if isinstance(wcjob_res,type({})):
+                jobreport = wcjob_res.get("jobreport")
+                wc_rst_list = wcjob_res.get("wc_rst_list")
+                taskid = jobreport.get("taskid")
+                jobid = jobreport.get("jobid")
+                logtype = jobreport.get("logtype")
+                received_at = jobreport.get("received_at")
+                starttime = jobreport.get("starttime")
+                endtime = jobreport.get("endtime")
+                total = jobreport.get("total")
+                isalert = jobreport.get("isalert")
+                pagecount = jobreport.get("pagecount")
+                keywordcount = jobreport.get("keywordcount")
+                hiddencount = jobreport.get("hiddencount")
+                tampcount = jobreport.get("tampcount")
+
+                trobj,flag = cm.ContentTasksResult.objects.get_or_create(
+                        task_id = task_id, 
+                        taskid = taskid,
+                        siteid = siteid,
+                        jobid = jobid,
+                        website_id = website_id
+                    )
+                trobj.logtype = logtype
+                trobj.received_at = received_at
+                trobj.starttime = starttime
+                trobj.endtime = endtime
+                trobj.total = total
+                trobj.isalert = isalert
+                trobj.keywordcount = keywordcount
+                trobj.hiddencount = hiddencount
+                trobj.tampcount = tampcount
+                trobj.wc_rst_list = json.dumps(wc_rst_list)
+                trobj.save()
+        except:
+            continue
+        
+
+def main():
+    """
+    """
+    collect_results()
+
+
+if __name__ == "__main__":
+    main()

src/script/start_collect_mtask_result_poc.py

@@ -0,0 +1,54 @@
+#-*-coding:utf-8 -*-
+import os,sys
+import django
+
+sys.path.append(os.path.join("..",os.path.dirname(__file__)))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+django.setup()
+
+import common.models as cm
+import webray_service.control_wshellpoc as cwsp
+
+
+def get_available_tasks():
+    """
+    """
+    wrtask = cm.WebRayTasks.objects.filter(type=100)
+    return wrtask
+
+
+def collect_results_poc():
+    tasks = get_available_tasks()
+    for task in tasks:
+        siteid = task.siteid
+        task_id = str(task.id)
+        website_id = task.website_id
+        wajob_res = cwsp.get_webshellpoc_results([{"task_id":task_id,"job_id":0}])
+        print wajob_res,44444444444444
+        #wa_rst_http_list = wajob_res.get("wa_rst_http_list")
+        #wa_rst_http_list = sorted(wa_rst_http_list,key=lambda x:x["received_at"],reverse=True)
+        #for rst in wa_rst_http_list:
+        #    taskid = rst.get("taskid")
+        #    jobid = rst.get("jobid")
+        #    logtype = rst.get("logtype")
+        #    httptime = rst.get("httptime")
+        #    received_at = rst.get("received_at")
+        #    isalert = rst.get("isalert")
+        #    trobj,flag = cm.AvailableTasksResult.objects.get_or_create(
+        #            task_id = task_id, 
+        #            taskid = taskid,
+        #            siteid = siteid,
+        #            jobid = jobid,
+        #            isalert = isalert,
+        #            website_id = website_id
+        #        )
+        #    trobj.logtype = logtype
+        #    trobj.httptime = httptime
+        #    trobj.received_at = received_at
+        #    trobj.save()
+        
+def main():
+    collect_results_poc()
+
+if __name__ == "__main__":
+    main()

src/script/start_collect_mtask_result_webshell.py

@@ -0,0 +1,74 @@
+#-*-coding:utf-8 -*-
+import json
+import os,sys
+import django
+
+sys.path.append(os.path.join("..",os.path.dirname(__file__)))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+django.setup()
+
+import common.models as cm
+import webray_service.control_wshellpoc as cwsp
+
+
+def get_available_tasks():
+    """
+    """
+    wrtask = cm.WebRayTasks.objects.filter(type=3)
+    return wrtask
+
+
+def collect_results_webshell():
+    tasks = get_available_tasks()
+    for task in tasks:
+        siteid = task.siteid
+        task_id = str(task.id)
+        website_id = task.website_id
+        wstrobj = cm.WebShellTasksResult.objects.filter(task_id=task.id).order_by("-jobid").first()
+        jobid = wstrobj.jobid if wstrobj else -1
+        if task.excute_type == "once":
+            if jobid > -1:
+                continue
+            else:
+                jobid = 0
+        else:
+            jobid = jobid + 1
+        wajob_res = cwsp.get_webshellpoc_results([{"task_id":task_id,"job_id":jobid}])
+        print wajob_res,2222222222
+        if wajob_res.get("message") == "ok":
+            if wajob_res.get("data"):
+                data = wajob_res.get("data")[0]
+                trobj,flag = cm.WebShellTasksResult.objects.get_or_create(
+                        task_type = 4,
+                        task_id = task.id, 
+                        website_id = website_id, 
+                        taskid = task.id,
+                        jobid = jobid,
+                        task_status = data.get("task_status"),
+                        starttime = data.get("task_begintime"),
+                        endtime = data.get("task_endtime"),
+                        task_executing_engine_id = data.get("task_executing_engine_id"),
+                        task_executing_enginevendor_id = data.get("task_executing_enginevendor_id"),
+                        task_result = json.dumps(data.get("task_result")),
+                        total = len(data.get("task_result",[]))
+                    )
+                trobj.save()
+                #生成webshell告警事件
+                if json.loads(trobj.task_result):
+                    cm.WarningEvent.objects.create(**{"event_type":"webshell",
+                            "task_type":1,
+                            "task_id":task_id,
+                            "taskid":task_id,
+                            "jobid":jobid,
+                            "taskname":task.name,
+                            "domain":task.site.split("//")[1].split(":")[0],
+                            "label":task.website_name,
+                            "organization":cm.WebSite.objects.filter(id=website_id).first().organization_name,
+                            "result":trobj.task_result
+                            })
+        
+def main():
+    collect_results_webshell()
+
+if __name__ == "__main__":
+    main()

src/settings/__init__.py

@@ -0,0 +1,3 @@
+#coding=utf-8
+from settings_test import *
+

src/settings/base.py

@@ -0,0 +1,132 @@
+#coding=utf-8
+"""
+Django settings for cloud_defense project.
+
+Generated by 'django-admin startproject' using Django 1.11.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.11/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = '19r&eh*ykwhtyg@zr0zvo(wnr_5y8qg5-1!0=a*!!$!pf3smi6'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = [
+    "*"
+]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'common',
+    'wsm',
+    'account',
+    'operation_log',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    #'django.middleware.cache.CacheMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'operation_log.OpLogMiddleware.OpLogMiddleware'
+]
+
+ROOT_URLCONF = 'urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [os.path.join(BASE_DIR,"../templates"),],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'wsgi.application'
+
+# Password validation
+# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.11/topics/i18n/
+
+LANGUAGE_CODE = 'zh-hans'
+
+TIME_ZONE = 'Asia/Shanghai'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = False
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.11/howto/static-files/
+
+STATIC_URL = '/static/'
+
+STATICFILES_DIRS = (
+    ("",os.path.join(BASE_DIR, '../static',"center")),
+)
+
+# 节点上报超时配置(单位:秒)
+SERVER_REPORT_TIMEOUT = 60 * 10
+
+AUTHENTICATION_BACKENDS = [
+    "account.cauth.AccountManage",
+]
+
+# 统计服务地址
+STATISTICS_SERVER = 'http://10.10.19.84:8090'

src/settings/settings_online.py

@@ -0,0 +1,80 @@
+# coding=utf-8
+from base import *
+
+# redis配置
+CACHES = {
+    'default': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': 'redis://127.0.0.1:6379/0',
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.default.DefaultClient",
+        },
+    },
+}
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'xeshop',
+        'USER': 'root',
+        'PASSWORD': 'xjc890*()',
+        'HOST': '127.0.0.1',
+        'PORT': '3306',
+    }
+}
+
+# 短信配置  common.sms.py使用
+SMS_ACCOUNT_SID = "8a216da8588b296f01588ecd0a0001ab"
+SMS_AUTH_TOKEN = "f0d2b53782374ebc8a76fa8dd06bb4d2"
+SMS_APP_ID = "8a216da8589ae4840158a89b5c8f0272"
+SMS_URL = "https://app.cloopen.com:8883/2013-12-26/Accounts/%s/SMS/TemplateSMS" % SMS_ACCOUNT_SID
+SMS_LOG_PATH = os.path.join(BASE_DIR, "../logs", "sms")
+SEND_SMS = False
+
+# 验证码字体库位置
+FONT_PATH = os.path.join(BASE_DIR, "../static/font", "consola.ttf")
+
+AUTHENTICATION_BACKENDS = [
+    "account.cauth.AccountManage",
+]
+
+# session 过期设置
+SESSION_COOKIE_AGE = 60 * 30
+SESSION_SAVE_EVERY_REQUEST = True
+SESSION_COOKIE_HTTPONLY = False
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s'
+        }
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'level': 'INFO',
+            'formatter': 'standard'
+        },
+    },
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
+        }
+    },
+}
+
+
+#channels                                                                          
+CHANNEL_LAYERS = {                                                                 
+    "default": {                                                                   
+        "BACKEND": "asgi_redis.RedisChannelLayer",                                     
+        "CONFIG": {                                                                    
+                    "hosts": [os.environ.get('REDIS_URL', 'redis://localhost:6379')],          
+                },                                                                             
+            "ROUTING": "dashboard.routings.channel_routing",                         
+        },                                                                             
+} 

src/settings/settings_test.py

@@ -0,0 +1,80 @@
+# coding=utf-8
+from base import *
+
+# redis配置
+CACHES = {
+    'default': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': 'redis://127.0.0.1:6379/0',
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.default.DefaultClient",
+        },
+    },
+}
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'xeshop',
+        'USER': 'root',
+        'PASSWORD': 'xjc890*()',
+        'HOST': '127.0.0.1',
+        'PORT': '3306',
+    }
+}
+
+# 短信配置  common.sms.py使用
+SMS_ACCOUNT_SID = "8a216da8588b296f01588ecd0a0001ab"
+SMS_AUTH_TOKEN = "f0d2b53782374ebc8a76fa8dd06bb4d2"
+SMS_APP_ID = "8a216da8589ae4840158a89b5c8f0272"
+SMS_URL = "https://app.cloopen.com:8883/2013-12-26/Accounts/%s/SMS/TemplateSMS" % SMS_ACCOUNT_SID
+SMS_LOG_PATH = os.path.join(BASE_DIR, "../logs", "sms")
+SEND_SMS = False
+
+# 验证码字体库位置
+FONT_PATH = os.path.join(BASE_DIR, "../static/font", "consola.ttf")
+
+AUTHENTICATION_BACKENDS = [
+    "account.cauth.AccountManage",
+]
+
+# session 过期设置
+SESSION_COOKIE_AGE = 60 * 30
+SESSION_SAVE_EVERY_REQUEST = True
+SESSION_COOKIE_HTTPONLY = False
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s'
+        }
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'level': 'INFO',
+            'formatter': 'standard'
+        },
+    },
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
+        }
+    },
+}
+
+
+#channels                                                                          
+CHANNEL_LAYERS = {                                                                 
+    "default": {                                                                   
+        "BACKEND": "asgi_redis.RedisChannelLayer",                                     
+        "CONFIG": {                                                                    
+                    "hosts": [os.environ.get('REDIS_URL', 'redis://localhost:6379')],          
+                },                                                                             
+            "ROUTING": "dashboard.routings.channel_routing",                         
+        },                                                                             
+} 

src/urls.py

@@ -0,0 +1,40 @@
+#coding=utf-8
+"""cloud_defense URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.11/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.conf.urls import url, include
+    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
+"""
+import os
+
+from django.conf import settings
+from django.conf.urls import url, include
+from django import views
+
+
+import common.views as cview
+import wsm.views as wview
+
+
+urlpatterns = [
+    url(r'^api/account/', include('account.urls_backstage')),
+    url(r'^api/doc/', wview.DocView.as_view()),
+    url(r'^api/hook/$', wview.HookView.as_view()),
+    url(r'^api/dashboard/', include('dashboard.urls_backstage')),
+    url(r'^api/oplog/', include('operation_log.urls_backstage')),
+]
+
+#urlpatterns += [
+#    url(r'^static/(?P<path>.*)$', views.static.serve, { 'document_root': settings.STATIC_ROOT})
+#]
+
+

src/utils/aestool.py

@@ -0,0 +1,98 @@
+#-*-coding:utf-8 -*-
+from cryptography.hazmat.primitives import padding
+from cryptography.hazmat.primitives.ciphers import algorithms
+from Crypto.Cipher import AES
+from binascii import b2a_hex, a2b_hex
+from urllib import urlencode
+from collections import OrderedDict
+import base64
+import json
+
+class AESCBCCrypt(object):
+
+    def __init__(self, key='c1a5fd9f724143e8'):
+        self.key = key.encode('utf-8')
+        self.mode = AES.MODE_CBC
+        self.iv = b'81dc198a449eb47f'
+        # block_size 128位
+
+    # 加密函数，如果text不足16位就用空格补足为16位，
+    # 如果大于16但是不是16的倍数，那就补足为16的倍数。
+    def encrypt(self, text):
+        cryptor = AES.new(self.key, self.mode, self.iv)
+        text = text.encode('utf-8')
+
+        # 这里密钥key 长度必须为16（AES-128）,24（AES-192）,或者32 （AES-256）Bytes 长度
+        # 目前AES-128 足够目前使用
+
+        text=self.pkcs7_padding(text)
+
+        self.ciphertext = cryptor.encrypt(text)
+
+        # 因为AES加密时候得到的字符串不一定是ascii字符集的，输出到终端或者保存时候可能存在问题
+        # 所以这里统一把加密后的字符串转化为16进制字符串
+        #return b2a_hex(self.ciphertext).decode().upper()
+        return base64.b64encode(self.ciphertext)
+
+    @staticmethod
+    def pkcs7_padding(data):
+        if not isinstance(data, bytes):
+            data = data.encode()
+
+        padder = padding.PKCS7(algorithms.AES.block_size).padder()
+
+        padded_data = padder.update(data) + padder.finalize()
+
+        return padded_data
+
+    @staticmethod
+    def pkcs7_unpadding(padded_data):
+        unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder()
+        data = unpadder.update(padded_data)
+
+        try:
+            uppadded_data = data + unpadder.finalize()
+        except ValueError:
+            raise Exception('无效的加密信息!')
+        else:
+            return uppadded_data
+
+    # 解密后，去掉补足的空格用strip() 去掉
+    def decrypt(self, text):
+        #  偏移量'iv'
+        cryptor = AES.new(self.key, self.mode, self.iv)
+        plain_text = cryptor.decrypt(a2b_hex(text))
+        # return plain_text.rstrip('\0')
+        return bytes.decode(plain_text).rstrip("\x01").\
+            rstrip("\x02").rstrip("\x03").rstrip("\x04").rstrip("\x05").\
+            rstrip("\x06").rstrip("\x07").rstrip("\x08").rstrip("\x09").\
+            rstrip("\x0a").rstrip("\x0b").rstrip("\x0c").rstrip("\x0d").\
+            rstrip("\x0e").rstrip("\x0f").rstrip("\x10")
+
+    def dict_json(self, d):
+        '''python字典转json字符串, 去掉一些空格'''
+        j = json.dumps(d).replace('": ', '":').replace(', "', ',"').replace(", {", ",{")
+        return j
+
+
+def genCode(*addargs,**kwargs):
+    """请求code生成
+    """
+    asst = {
+        "appkey":1010,
+        "secret":"00660C74B2270D76",
+        "session":"5C19205922F810589F164DF6A22A52AA",
+    }
+    args = ["appkey","session","secret"]
+    kwargs.update(asst)
+    args.extend(addargs)
+    codestr = ""
+    for k in args:
+        v = kwargs.get(k)
+        codestr += "{}={}&".format(k,v)
+    codestr = codestr.rstrip("&")
+    codestr = AESCBCCrypt().encrypt(codestr)
+    codestr = urlencode({"code":codestr})
+    return codestr
+
+

src/utils/aliyun_email.py

@@ -0,0 +1,97 @@
+#-*-coding:utf-8 -*-
+import hmac
+import base64
+import requests
+import datetime,time
+import urllib
+import uuid
+from hashlib import sha1
+
+class AliEmailClient(object):
+    """阿里云邮件推送
+    """
+    def __init__(self):
+        self.url = "http://dm.aliyuncs.com"
+        self.AccessKeyId = "LTAI6puCBe6A1lOa"
+        self.AccessKeySecret = "VeWxSWEWM8xULzbnbCHDkSn7QPvasO"
+
+    def percent_encode(self,encodeStr):
+        """
+        """
+        encodeStr = str(encodeStr)
+        res = urllib.quote(encodeStr.decode("utf-8").encode("utf-8"),"")
+        res = res.replace("+","%20")
+        res = res.replace("*","%2A")
+        res = res.replace("%7E","~")
+
+        return res
+
+    def gen_signature(self,params):
+        """
+        """
+        sortedParams = sorted(params.items(),key=lambda x:x[0])
+        canonicalizedQueryString = ''
+        for k,v in sortedParams:
+            canonicalizedQueryString += '&' + self.percent_encode(k) \
+                + '=' + self.percent_encode(v)
+        stringToSign = 'GET&%2F&' + self.percent_encode(canonicalizedQueryString[1:])
+        hmc = hmac.new(self.AccessKeySecret + "&",stringToSign,sha1)
+        signature = base64.encodestring(hmc.digest()).strip()
+        
+        return signature
+
+    def gen_params(self,params={}):
+        """
+        """
+        pub_param = {
+            "Format":"JSON", 
+            "Version":"2015-11-23",
+            "AccessKeyId":self.AccessKeyId,
+            "SignatureMethod":"HMAC-SHA1",
+            "Timestamp":time.strftime("%Y-%m-%d %H:%M:%S",time.gmtime()),
+            "SignatureVersion":"1.0",
+            "SignatureNonce":str(uuid.uuid1()),
+        }
+        for key in pub_param:
+            params.update({key:pub_param[key]})
+        signature = self.gen_signature(params)
+        params["Signature"] = signature
+
+        return params
+        
+
+    def send_mail(self,to,subject,content):
+        """
+        """
+        params = {
+            "Action":"SingleSendMail",
+            "AccountName":"swad-services@mail.365gcd.com",
+            "ReplyToAddress":True,
+            "AddressType":1,
+            "ToAddress":to,
+            "FromAlias":"中国网安云安全".decode("utf-8").encode("utf-8"),
+            "Subject":unicode(subject).encode("utf-8"),
+            "HtmlBody":unicode(content).encode("utf-8")
+        }
+        url = self.url + "/?" + urllib.urlencode(self.gen_params(params))
+        res = requests.get(url)
+        print res
+        return res.json() if res else {}
+
+    def batch_send_mail(self,to,subject,content):
+        """批量发送
+        """
+if __name__ == "__main__":
+    aliemail_client = AliEmailClient()
+    to = "scxiaojincai@163.com"
+    to = "496498291@qq.com"
+    subject = u"测试"
+    content = u"测试"
+    res = aliemail_client.send_mail(to,subject,content)
+    print res
+
+
+
+
+
+

src/utils/cloopen_sms.py

@@ -0,0 +1,164 @@
+#coding=utf-8
+'''
+Created on 2018年12月11日
+@author: xiaojincai
+'''
+import os
+import json
+import random
+import datetime
+from hashlib import md5
+import threading
+import logging
+
+import base64
+import requests
+
+logger = logging.getLogger(__name__)
+
+SMS_TEMPLATE_DCT = {
+    "version":"v2.4",
+    "sign":u"【网安云防御】",
+    "396732":{
+        "template_id":"396732",
+        "title":u"用户信息邮件变更通知",
+        "content":u"您的注册邮箱地址在{}由{}变更为{}。如有疑问，请及时与运营人员联系！" 
+    },
+    "400033":{
+        "template_id":"400033",
+        "title":u"用户信息手机号变更通知", 
+        "content":u"您的注册手机号码在{}由{}变更为{}。如有疑问，请及时与运营人员联系！" 
+    },
+    "396735":{
+        "template_id":"396735",
+        "title":u"账号到期提醒通知", 
+        "content":u"您的账号{}将于{}到期，届时该账号下的网站将被停止安全防护服务，请及时与运营人员联系！" 
+    },
+    "396737":{
+        "template_id":"396737",
+        "title":u"账号到期提醒通知(运营)", 
+        "content":u"{}账号将于{}到期，请及时处理！" 
+    },
+    "396739":{
+        "template_id":"396739",
+        "title":u"网站迁出提醒通知(用户)", 
+        "content":u"经检测发现，您的网站{}在{}已迁出云防御系统，处于未防御状态，请予以关注！" 
+    },
+    "396740":{
+        "template_id":"396740",
+        "title":u"网站迁出提醒通知(运营)", 
+        "content":u"经检测发现，网站{}在{}已迁出云防御系统，处于未防御状态，请及时处理！" 
+    },
+    "396742":{
+        "template_id":"396742",
+        "title":u"防御状态变更通知(用户)", 
+        "content":u"您的网站{}在{}变更为{}状态，请予以关注！" 
+    },
+    "396744":{
+        "template_id":"396744",
+        "title":u"防御状态变更通知(运营)", 
+        "content":u"网站{}在{}变更为{}状态，请及时处理！" 
+    },
+    "396746":{
+        "template_id":"396746",
+        "title":u"网站删除通知", 
+        "content":u"网站{}在{}被用户{}删除，云防御系统将无法继续提供安全防护服务，请予以关注！" 
+    },
+    "397198":{
+        "template_id":"397198",
+        "title":u"网站防护策略修改通知", 
+        "content":u"网站{}在{}被用户{}修改了{}策略，请予以关注！" 
+    },
+    "400981":{
+        "template_id":"400981",
+        "title":u"网站到期提醒", 
+        "content":u"您的网站{}将于{}到期，届时该网站将会被自动回源，停止安全防护服务，请及时与运营人员联系！" 
+    },
+    "400984":{
+        "template_id":"400984",
+        "title":u"网站到期提醒(运营)", 
+        "content":u"{}网站将于{}到期，请及时处理！" 
+    },
+    "401151":{
+        "template_id":"401151",
+        "title":u"重置密码短信验证码", 
+        "content":u"您正在进行重置密码操作，验证码是{}，请于{}分钟内正确输入!" 
+    },
+    "423122":{
+        "template_id":"423122",
+        "title":u"网站可用状态变更", 
+        "content":u"您的网站{}在{}{}为{}状态，请予以关注！" 
+    }
+}
+
+class CloopenSMS(object):
+    """融联短信接口
+    """
+    def __init__(self):
+        self.sid = "8a216da8588b296f01588ecd0a0001ab"
+        self.token = "f0d2b53782374ebc8a76fa8dd06bb4d2"
+        self.appId = "8a216da8589ae4840158a89b5c8f0272"
+        self.base_url = "https://app.cloopen.com:8883/2013-12-26/Accounts/%s/SMS/TemplateSMS" % self.sid
+         
+    def send(self,phones,template_id,content):
+        """
+        @attention: 发送短信
+        @param content: 短信内容模板变量数组
+        @param phones: 电话号码集合,格式为数组
+        """
+        #构造url
+        now = datetime.datetime.now()
+        batch = now.strftime("%Y%m%d%H%M%S")
+        signature = self.sid + self.token + batch
+        sig = md5(signature).hexdigest().upper()
+        url = self.base_url + "?sig=" + sig
+        
+        #构造header
+        src = "%s:%s"%(self.sid,batch)
+        auth = base64.encodestring(src).strip()
+        headers = {"Accept": "application/json",
+                   "Content-Type": "application/json;charset=utf-8",
+                   "Authorization": auth}
+        
+        #构造body
+        content = map(lambda x:x if isinstance(x, unicode) else str(x).decode("utf8"), content)
+
+        print "send sms >>>:",phones,template_id,content
+        body = {"to": ','.join(phones),
+                "appId": self.appId,
+                "templateId": template_id,
+                "datas": content
+                }
+            
+        #成功直接返回1,失败记录日志,并返回0
+        status_code = ""
+        status_msg = ""
+        response = requests.post(url, headers=headers, data=json.dumps(body))
+        response_code = response.status_code
+        if response_code==200:
+            data = response.json()
+            if data["statusCode"]=="000000":
+                return 1,data
+            else:
+                status_code = data["statusCode"]
+                status_msg = data["statusMsg"]
+                return 0,data
+                
+        #内容,模板ID,号码,http错误码,定义的错误码,操作时间,url
+        now_time = now.strftime("%Y-%m-%d %H:%M:%S")
+        msg = "<<(%s_%s_%s) %s>>:url=%s;\n"%(response_code,status_code,status_msg,now_time,url)
+        sc = ",".join(content)
+        sp = ",".join(phones)
+        msg += "content=[%s];template_id=%s;phones=[%s]\n"%(sc,template_id,sp)
+        return 0,u"未知错误(短信接口报错)"
+
+cloopensms = CloopenSMS()
+
+if __name__ == '__main__':
+    #测试
+    phones = ['15982456282']
+    template_id = '396732'
+    content = ["2018-12-11 14:00:00","496498291@qq.com","496498291@qq.com"]
+
+    cloopensms.send(phones,template_id,content)
+

src/utils/constant.py

@@ -0,0 +1,29 @@
+# coding:utf-8
+# 常用的常量
+DATETIME_FORMAT = "%Y%m%d%H%M%S"
+DATE_FORMAT = "%Y%m%d"
+TIME_FORMAT = "%H%M%S"
+
+DATETIME_FORMAT_DEFAULT = "%Y-%m-%d %H:%M:%S"
+DATE_FORMAT_DEFAULT = "%Y-%m-%d"
+TIME_FORMAT_DEFAULT = "%H:%M:%S"
+
+DATE_FORMAT_SPOT = "%Y.%m.%d"
+
+EVENT_TYPE_CHO = (
+    (1,u"可用性事件"),
+    (2,u"篡改事件"),
+    (3,u"敏感词事件"),
+    (4,u"非法外链事件"),
+    (5,u"非法外链事件"),
+    (6,u"漏洞事件")
+)
+
+EVENT_TYPE_DCT = {
+    "avaible":u"可用性事件",
+    "tamp":u"篡改事件",
+    "keyword":u"敏感词事件",
+    "hidden":u"非法外链事件",
+    "bug":u"漏洞事件",
+    "webshell":u"webshell"
+}

src/utils/email_client.py

@@ -0,0 +1,134 @@
+# coding:utf-8
+# !/user/bin/python2
+"""
+邮件发送模块
+"""
+import time
+import json
+import logging
+from functools import partial
+import threading
+import requests
+import traceback
+
+from email.header import Header
+from email.MIMEText import MIMEText
+from email.utils import parseaddr,formataddr
+import smtplib,time,os
+
+from aliyun_email import AliEmailClient
+
+
+logger = logging.getLogger(__name__)
+
+
+
+Settings = type('Settings', (object, ), 
+                {'EMAIL_URL': 'https://182.138.102.104/push-gateway/v1/push', 
+                 'EMAIL_USER': 'mail.cetcsc.com',
+                 'EMAIL_PASSWD': '30sanpush'})
+
+
+class EmailClient(object):
+    """
+    邮件客户端
+    """
+
+    def __init__(self, url, user, passwd):
+        self.url = url
+        self.user = user
+        self.passwd = passwd
+
+    #def send(self, to, subject, content):
+    #    """
+    #    发送邮件
+    #    :params
+    #    to:   邮件列表 
+    #    subject: 主题
+    #    content: 内容
+    #    """
+    #    auth = requests.auth.HTTPDigestAuth(self.user, self.passwd)
+    #    data = self._gen_content(to, subject, content)
+    #    logger.info("send email, raw data: %s", str(data))
+    #    try:
+    #        r = requests.post(self.url, json=data, auth=auth, verify=False)
+    #        print r.json()
+    #    except Exception as e:
+    #        print e
+    #        return False,str(e)
+    #    if r.status_code != requests.codes.OK:
+    #        return False,r.json()
+    #    try:
+    #        logger.info('send email response: %s', str(r.text))
+    #        ret = r.json()
+    #        if ret['mail']['code'] != '0':
+    #            logger.error("send email failed, reason: %s ", str(ret))
+    #            return False,ret
+    #    except Exception as e:
+    #        return False,str(e)
+    #    return True,"success"
+
+    #def send(self,to,subject,content):
+    #    """
+    #    """
+    #    from_addr = "cetcsc_swad@163.com"
+    #    password = "AXN4zwIoCd"
+    #    smtp_server = "smtp.163.com"
+    #    #自定义处理邮件收发地址的显示内容
+    #    def _format_addr(s):
+    #        name,addr = parseaddr(s)
+    #        #将邮件的name转换成utf-8格式，addr如果是unicode，则转换utf-8输出，否则直接输出addr
+    #        return formataddr((\
+    #            Header(name,'utf-8').encode(),\
+    #            addr.encode("utf-8") if isinstance(addr,unicode) else addr))
+
+    #    #邮件正文是MIMEText
+    #    msg = MIMEText(content,'html','utf-8')
+    #    #邮件对象
+    #    msg['From'] = _format_addr(u'网络监测预警事业部 <%s>'%from_addr)
+    #    #msg['to'] = _format_addr(u'管理员<%s>'%to)
+    #    msg['Subject'] = Header(subject,'utf-8').encode()
+    #    msg['date'] = time.strftime("%a,%d %b %Y %H:%M:%S %z")
+    #    #发送邮件
+    #    try:
+    #        server = smtplib.SMTP(smtp_server,25)
+    #        #server.set_debuglevel(1)
+    #        server.login(from_addr,password)
+    #        server.sendmail(from_addr,to,msg.as_string())
+    #        server.quit()
+    #    except Exception as e:
+    #        return False,str(e)
+    #    return True,"success"
+
+    def send(self,to,subject,content):
+        """
+        """
+        aliemail_client = AliEmailClient()
+        to = to[0] if isinstance(to,type([])) else to
+        res = aliemail_client.send_mail(to,subject,content)
+        if res and res.get("RequestId") and res.get("EnvId"):
+            return True,"success"
+        else:
+            return False,res.get("Message","fail") if res else "fail"
+
+
+    def _gen_content(self, to, subject, content):
+        data = {
+                    "emails": to,
+                    "title": subject,
+                    "body": {
+                        "textHTML": content
+                    }
+                }
+        return data
+
+
+email_client = EmailClient(Settings.EMAIL_URL, Settings.EMAIL_USER, Settings.EMAIL_PASSWD)
+
+if __name__ == '__main__':
+    #测试
+    to = ["scxiaojincai@163.com"]
+    to = "496498291@qq.com"
+    subject = u"删除提醒"
+    content = u"网站{www.test1.com}在2018-08-30 15:30:10 被用户{xxx}删除，云防御系统将无法继续提供安全防护服务，请予以关注！"
+    email_client.send(to,subject,content)

src/utils/excel_output.py

@@ -0,0 +1,33 @@
+# coding:utf-8
+import StringIO
+import xlwt
+import zipfile
+import datetime
+import random, string
+import os
+from ftplib import FTP
+
+def save2memory(en_cn_map, content):
+    """
+    保存内容到xls里面
+    :param en_cn_map:  英文中文对应字典 {"name": "名字"}, OrderedDict
+    :param content:
+    :return:
+    """
+    wb = xlwt.Workbook(encoding="utf-8")
+    sheet = wb.add_sheet('info')
+    sort_head = en_cn_map.keys()
+    for i, item in enumerate(en_cn_map):
+        sheet.write(0, i, en_cn_map.get(item))
+    for row, item in enumerate(content):
+        sort_key = item.keys()
+        for key in sort_key:
+            if key not in sort_head:
+                continue
+            sheet.write(row+1, sort_head.index(key), item.get(key, ''))
+    output = StringIO.StringIO()
+    wb.save(output)
+    return output.getvalue()
+
+if __name__ == '__main__':
+    pass

src/utils/exceltool.py

@@ -0,0 +1,33 @@
+#-*-coding:utf-8 -*-
+import StringIO
+import xlwt
+
+class ExcelTool(object):
+    """
+    """
+    def __init__(self,filename=None):
+        self.filename = filename
+        self.book = xlwt.Workbook(encoding="utf-8",style_compression=0)
+
+    def save_data(self,sheet,header,data):
+        """
+        """
+        sheet = self.book.add_sheet(sheet,cell_overwrite_ok=True)
+        #添加表头
+        for i in range(0,len(header)):
+            sheet.write(0,i,header[0])
+
+        #添加数据
+        for i in range(0,len(data)):
+            for j in range(0,len(data[i])): 
+                sheet.write(i+1,j,data[i][j])
+        output = StringIO.StringIO()
+        self.book.save(output)
+        self.book.save("/tmp/demo.xls")
+        return output.getvalue()
+
+
+#if __name__ == "__main__":
+#    mexcel = ExcelTool("/tmp/demo.xls")
+#    mexcel.save_data(u"测试",[u"事件类型"],[[u"漏洞事件"]])
+

src/utils/website_info_collect/Logger.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# author: liangbo
+# date:2014-03-18
+
+#
+# GLOBAL IMPORTS
+#
+import string, logging, os
+import logging
+#from logging.handlers import SysLogHandler
+
+# syslogger = logging.getLogger('syslog')
+# syslogger.setLevel(logging.INFO)
+#
+# syslog_fmt = "365GCD: "
+# fomatter = logging.Formatter(syslog_fmt)
+# syslog_handler = SysLogHandler(('10.10.19.82', 514), facility='local7')
+# syslog_handler.setFormatter(fomatter)
+# syslogger.addHandler(syslog_handler)
+#
+#
+# logger = logging.getLogger('machine_learn_detect')
+# logger.setLevel(logging.INFO)
+# stream_fmt = "%(asctime)s %(module)s %(funcName)s [%(levelname)s]: %(message)s"
+# fomatter = logging.Formatter(stream_fmt)
+# streamhandler = logging.StreamHandler()
+# streamhandler.setFormatter(fomatter)
+# logger.addHandler(streamhandler)
+
+class Logger:
+    '''
+        Static class for logging purposes
+         Use from other classes:
+
+           from Logger import Logger
+           logger = Logger.logger
+
+           logger.debug("Some debug")
+           logger.info("Some info")
+           logger.error("Error")
+        More info at http://docs.python.org/lib/module-logging.html
+    '''
+
+    logger = logging.getLogger('machine_learn_detect')
+    logger.setLevel(logging.INFO)
+
+    DEFAULT_FORMAT = '%(asctime)s %(module)s %(funcName)s [%(levelname)s]: %(message)s'
+    SYSLOG_FORMAT = '%(message)s'
+    __formatter = logging.Formatter(DEFAULT_FORMAT)
+    __streamhandler = None
+
+    # load the console handler by default
+    # it will be removed in daemon mode
+    __streamhandler = logging.StreamHandler()
+    __streamhandler.setFormatter(__formatter)
+    logger.addHandler(__streamhandler)
+
+    # Removes the stream handler
+    # (Useful when agent starts in daemon mode)
+    def remove_console_handler():
+        if Logger.__streamhandler:
+            Logger.logger.removeHandler(Logger.__streamhandler)
+
+    remove_console_handler = staticmethod(remove_console_handler)
+
+    # log to file (file should be log->file in configuration)
+    @staticmethod
+    def _add_file_handler(file, log_level=None):
+
+        dir = file.rstrip(os.path.basename(file))
+        if not os.path.isdir(dir):
+            try:
+                os.makedirs(dir, 0755)
+            except OSError, e:
+                print "Logger: Error adding file handler,", \
+                    "can not create log directory (%s): %s" % (dir, e)
+                return
+
+        try:
+            handler = logging.FileHandler(file)
+        except IOError, e:
+            print "Logger: Error adding file handler: %s" % (e)
+            return
+
+        handler.setFormatter(Logger.__formatter)
+
+        if log_level:  # modify log_level
+            handler.setLevel(log_level)
+        Logger.logger.addHandler(handler)
+
+
+    @staticmethod
+    def add_file_handler(file):
+        Logger._add_file_handler(file)
+
+
+    # Error file handler
+    # the purpouse of this handler is to only log error and critical messages
+    @staticmethod
+    def add_error_file_handler(file):
+        Logger._add_file_handler(file, logging.ERROR)
+
+
+    # send events to a remote syslog
+    @staticmethod
+    def add_syslog_handler(address):
+        from logging.handlers import SysLogHandler
+        handler = SysLogHandler(address, facility='local7')
+        handler.setFormatter(logging.Formatter(Logger.SYSLOG_FORMAT))
+        Logger.logger.addHandler(handler)
+
+
+
+    # show DEBUG messages or not
+    # modifying the global (logger, not handler) threshold level
+    def set_verbose(verbose='info'):
+        if verbose.lower() == 'debug':
+            Logger.logger.setLevel(logging.DEBUG)
+        elif verbose.lower() == 'info':
+            Logger.logger.setLevel(logging.INFO)
+        elif verbose.lower() == 'warning':
+            Logger.logger.setLevel(logging.WARNING)
+        elif verbose.lower() == 'error':
+            Logger.logger.setLevel(logging.ERROR)
+        elif verbose.lower() == 'critical':
+            Logger.logger.setLevel(logging.CRITICAL)
+        else:
+            Logger.logger.setLevel(logging.INFO)
+
+    set_verbose = staticmethod(set_verbose)
+
+    def next_verbose_level(verbose):
+        levels = ['debug', 'info', 'warning', 'error', 'critical']
+        if verbose in levels:
+            index = levels.index(verbose)
+
+            if index > 0:
+                return levels[index - 1]
+
+        return verbose
+
+    next_verbose_level = staticmethod(next_verbose_level)
+
+
+if __name__ == "__main__":
+    logger = Logger.logger
+    Logger.set_verbose('debug')
+    Logger.add_syslog_handler(("10.4.5.86", 514))
+
+    # logs to console
+    logger.debug("Some debug text")
+    logger.info("365GCD: Some info text==============================")
+    logger.critical("Oppps, error")
+
+    # now logs to file and not to log
+    # Logger.add_file_handler('/tmp/agent.log')
+    # Logger.add_error_file_handler('/tmp/agent-error.log')
+    # Logger.remove_console_handler()
+    # logger.debug("log debug info to file")
+    # logger.warning("log warning info to file")
+    # logger.error("log error info to file")
+

src/utils/website_info_collect/Readme.md

@@ -0,0 +1,45 @@
+网站信息查询工具
+=============================
+
+提供网站ICP备案信息查询，网站子域名及IP、IP地域、IP接入商、CDN/WAF检测、虚拟空间检测等信息。
+
+--------------------------------
+# 运行环境
+python==2.7,其它依赖库见requirements.txt
+
+# 快速开始
+## 配置
+```bash
+config.py中
+配置ICP接口keys列表，初始key提供35183次查询
+配置'PROCESS_POOL_SIZE',设置进程池大小，默认8
+```
+## 命令
+```bash
+python main.py -n 365gcd.com #单域名方式、365gcd.com为要查询的域名 
+python main.py -f domains.txt  #批量域名方式、domain.txt为域名列表，一行一个域名，输出为outfile_时间.csv
+```
+# 功能指标
+## 功能概览
+* 已完成的信息：主办单位性质、是否备案、ICP备案号、
+* 部分完成的信息：虚拟空间识别、安全设备判断、安全设备识别、CDN判断、CDN识别
+* 未完成的信息：网站负责人
+
+## CDN详情
+* 已收录:百度云加速
+* 计划收录: Ucloud、 网宿、阿里云、腾讯云、金山云
+
+## WAF详情
+* 已收录: 创宇盾
+* 计划收录: 安全狗、 360安域、 阿里云WAF、 腾讯云WAF、安恒WAF、 有云WAF
+
+## 虚拟空间详情
+* 已收录: 新网虚拟主机、万网虚拟主机、西部数码虚拟主机、美橙科技、百度虚拟空间
+* 计划收录: 华为云、35互联、新网互联、华夏名网、联动天下、景安数据、中国数据
+
+# 注意
+* 目前不支持中文域名查询，因为chinaz的接口不支持。
+
+
+
+

src/utils/website_info_collect/beian.py

@@ -0,0 +1,377 @@
+# -*- coding:utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     icp_check
+   Description :   网站信息查询
+   Author :       lihanrui
+   date：          2019/3/4
+-------------------------------------------------
+   Change Activity:
+                   2019/3/4:
+-------------------------------------------------
+"""
+import requests
+import json
+import socket
+import argparse
+import os
+import time
+import random
+import csv
+
+from multiprocessing import Pool
+socket.setdefaulttimeout(5)
+
+from ipip import ip_manager
+from config import *
+from Logger import Logger
+
+log = Logger.logger
+
+
+def read_file(fpath):
+    if not os.path.exists(fpath):
+        print "%s not exist!" % fpath
+        exit(1)
+    with open(fpath, 'r') as f:
+        return f.readlines()
+
+
+def save_file(filepath, content):
+    try:
+
+        with open(filepath, 'w') as f:
+            f.write(content)
+    except Exception as e:
+        log.debug("%s,%s" % (filepath, e))
+
+
+def http_get(url, headers={}):
+
+    retry = 0
+    headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"})
+
+    while retry < 2:
+        try:
+            response = requests.get(url, headers=headers, timeout=3)
+            response.encoding = 'utf-8'
+            return response.status_code, response.text
+        except Exception as e:
+            retry += 1
+            time.sleep(0.1)
+
+    return 'except', 'except'
+
+
+def check_access_protect(ip, domain):
+
+    ret = dict()
+    if len(ip) == 0:
+        return ret
+
+    access_url = "http://%s/" % ip
+    reps = []
+    for template in ["http://%s/?id=1 or 1=1",
+                     "http://%s/?id=../../etc/passwd",
+                     "http://%s/?id=http://aewrae.aliff.com/1.txt?%%00"]:
+
+        attack_url = template % ip
+        content, code = http_get(attack_url, {"Host": domain})
+        reps.append((content, code))
+
+    ret["access_rep_code"], _ = http_get(access_url, {"Host": domain})
+
+    if ret["access_rep_code"] in [200, 206, 301, 302, 404]:
+        ret["access_rep"] = '是'
+        for code, content in reps:
+            if code != 200 and code != 404:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '是'
+            else:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '否'
+    else:
+        ret["access_rep"] = '否'
+        for code, content in reps:
+            if code != ret["access_rep_code"]:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '是'
+            else:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '否'
+
+    return ret
+
+
+def check_virtualspace(ip, domain):
+    ret = dict()
+    if len(ip) == 0:
+        return ret
+
+    url = "http://%s/" % ip
+    code, content = http_get(url)
+    for i in VIRTUAL_SPACE.keys():
+        if content.find(i) == NOT_FOUND:
+            if len(content) != 0:
+                file_name = "%s_%s.html" % (ip, domain)
+                file_path = os.path.join(vir_dir, file_name)
+                save_file(file_path, content)
+        else:
+            ret["virtualspace"] = VIRTUAL_SPACE[i].encode("utf-8")
+
+    for i in CDN.keys():
+        if content.find(i) == NOT_FOUND:
+            continue
+        else:
+            ret["cdn_name"] = CDN[i].encode("utf-8")
+
+    return ret
+
+
+def get_domain_whois(domain):
+
+    def _get(url):
+        try:
+            response = requests.get(url, timeout=3)
+            return response.json()
+        except Exception as e:
+            log.exception("Request whois interface exception:%s" % e)
+        return {}
+
+    ret = dict()
+    domain = domain.strip()
+    if len(domain) == 0:
+        return ret
+
+    url = whois_interface % (random.choice(keys), domain)
+    json_rep = _get(url)
+    if json_rep:
+        status_code = json_rep.get("StateCode", 0)
+        if status_code == 1:
+            data = json_rep.get("Result", "")
+            if not data:
+                log.info(u"%s 暂无Whois信息..." % domain)
+                return ret
+            ret["ExpirationDate"] = data["ExpirationDate"].encode("utf-8")
+        else:
+            log.info(json.dumps(json_rep, ensure_ascii=False))
+    return ret
+
+def get_ip_info(domain):
+
+    def _get_ip(domain):
+        try:
+            myaddr = socket.getaddrinfo(domain, 'http')
+            return myaddr[0][4][0]
+        except Exception as e:
+            log.exception("%s ,%s" % (domain, e))
+            return None
+
+    def _get_ipip_info(ip):
+        ipx_res = ip_manager.find(ip)
+        if len(ipx_res):
+            country = ipx_res[0]
+            province = ipx_res[1]
+            city = ipx_res[2]
+            # 运营商
+            operator = ipx_res[4]
+            return country, province, city, operator
+        else:
+            return ['']*4
+
+    ret = dict()
+    ip = _get_ip(domain)
+    if ip:
+        country, province, city, operator = _get_ipip_info(ip)
+
+        ret["ip"] = ip
+        ret["ip_location"] = ''.join([country, province, city]).encode("utf-8")
+        ret["operator"] = operator.encode("utf-8")
+
+    return ret
+
+
+def get_website_icpinfo(domain):
+
+    domain = domain.strip()
+
+    def _get(url):
+        try:
+            response = requests.get(url, timeout=3)
+            return response.json()
+        except Exception as e:
+            log.exception("Request icp interface exception:%s" % e)
+        return {}
+    ret = dict()
+
+    url = icp_interface % (random.choice(keys), domain)
+
+    json_rep = _get(url)
+
+    if json_rep:
+        status_code = json_rep.get("StateCode", 0)
+        if status_code == 1:
+            data = json_rep.get("Result", "")
+            if not data:
+                log.info(u"%s 暂无备案信息..." % domain)
+                return ret
+            ret["CompanyName"] = data["CompanyName"].encode("utf-8")
+            ret["CompanyType"] = data["CompanyType"].encode("utf-8")
+            ret["MainPage"] = data["MainPage"]
+            ret["SiteLicense"] = data["SiteLicense"].encode("utf-8")
+            ret["SiteName"] = data["SiteName"].encode("utf-8")
+            ret["VerifyTime"] = data["VerifyTime"]
+        else:
+            log.info(json.dumps(json_rep, ensure_ascii=False))
+    return ret
+
+
+def check_expire_date(str_time):
+    try:
+        expire_date = time.strptime(str_time, '%Y-%m-%d %H:%M:%S')
+    except Exception as e:
+        return '不定'
+
+    now = time.time()
+    str_expire = time.mktime(expire_date)
+    if str_expire - now > 0:
+        return '否'
+    else:
+        return '是'
+
+
+
+
+
+
+
+def get_website_info(domain):
+
+    ret_infos = []
+
+    domain = domain.strip()
+
+    #网站IPC信息查询,没有子域名直接返回
+    ipc_info = get_website_icpinfo(domain)
+    mainpage = ipc_info.pop("MainPage", '')
+    if len(mainpage) == 0:
+        info = ['不定']*21
+        info[0] = domain
+        ret_infos.append(info)
+        return ret_infos
+
+    for subdomain in mainpage.split(" "):
+        #网站IP信息查询
+        ip_info = get_ip_info(subdomain)
+        #网站是否可访问、是否存在安全设备
+        access_info = check_access_protect(ip_info.get("ip", ''), subdomain)#(icp_info["MainPage"])
+        #网站是否在虚拟空间
+        virtualspace = check_virtualspace(ip_info.get("ip", ''), subdomain)
+        #域名whois信息(过期时间)
+        whois_info = get_domain_whois(subdomain)
+
+        ret_infos.append(
+            [domain.encode('utf-8'), subdomain.encode('utf-8'),
+             whois_info.get("ExpirationDate", '不定').strip(), check_expire_date(whois_info.get("ExpirationDate", '不定').strip()),
+             ipc_info.get("SiteName", '不定'), access_info.get("access_rep", '否'),
+             ip_info.get("ip", '不定'), ip_info.get("ip_location", '不定'), ip_info.get("operator", '不定'),
+             ipc_info.get("CompanyType", '不定'), ipc_info.get("CompanyName", '不定'),
+
+             '是' if ipc_info.get("SiteLicense", '') != "" else '否', ipc_info.get("SiteLicense", '不定'),
+             '',
+             '是' if virtualspace.get("virtualspace", '') != "" else '不定', virtualspace.get("virtualspace", '不定'),
+              access_info.get("safe_device", '不定'),
+             '是' if virtualspace.get("cdn_name", '') != "" else '不定', virtualspace.get("cdn_name", '不定'),
+
+             time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), '']
+        )
+    return ret_infos
+
+
+def output_csv(infos):
+
+    localtime = time.localtime()
+    strtime = time.strftime("%Y%m%d%H%M%S", localtime)
+    fname = "outfile_%s.csv" % strtime
+
+    headers = ["根域名", "子域名枚举",
+               "域名到期时间","域名是否到期",
+               "网站名称", "是否正常访问",
+               "所属IP", "地理位置", "运营商",
+               "主办单位性质", "主办单位名称",
+
+               "是否备案", "ICP备案号",
+               "备案时间",
+               "是否虚拟空间接入", "虚拟空间服务商",
+               "是否有安全防护",
+               "是否有CDN","CDN厂商",
+
+               "检查时间", "备注"]
+    with open(fname, 'wb') as f:
+        writer = csv.writer(f)
+        writer.writerow(headers)
+        for info in infos:
+            try:
+                writer.writerow(info)
+            except Exception as e:
+                log.exception("%s" % e)
+
+
+def init():
+
+    if not os.path.exists(os.path.dirname(LOG_File)):
+        os.mkdir(os.path.dirname(LOG_File))
+
+    if not os.path.exists(vir_dir):
+        os.mkdir(vir_dir)
+
+    Logger.add_file_handler(LOG_File)
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(prog="icp_info_check", usage="%(prog)s [options] usage", description="Website Information (Incloud: ICP Record Information) Query Tool")
+    parser.add_argument("-n", "--name", help="The root domain name you want to find")
+    parser.add_argument("-f", "--filepath", help="The root domain name list path")
+    parser.add_argument('-v', '--version', action='version', version='version 0.0.1')
+
+    args = parser.parse_args()
+
+    init()
+    if args.filepath:
+
+        filepath = args.filepath
+        lines = read_file(filepath)
+        infos = list()
+        p = Pool(PROCESS_POOL_SIZE)
+        l_result = p.map(get_website_info, lines)
+        for r in l_result:
+            infos.extend(r)
+        if len(infos) != 0:
+            output_csv(infos)
+        else:
+            print "No website parsing success!"
+
+    elif args.name:
+        headers = ["根域名", "子域名枚举",
+                   "域名到期时间","域名是否到期",
+                   "网站名称", "是否正常访问",
+                   "所属IP", "地理位置", "运营商",
+                   "主办单位性质", "主办单位名称",
+                   "是否备案", "ICP备案号",
+                   "备案时间",
+                   "是否虚拟空间接入", "虚拟空间服务商",
+                   "是否有安全防护",
+                   "是否有CDN", "CDN厂商",
+                   "检查时间", "备注"]
+        domain = args.name.strip()
+        info = get_website_info(domain)
+        format_info = dict(zip(headers, info[0]))
+        print json.dumps(format_info, ensure_ascii=False, indent=4).decode("utf-8").encode("gbk")
+
+
+
+
+
+
+
+

src/utils/website_info_collect/config.py

@@ -0,0 +1,60 @@
+# -*- coding:utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     config
+   Description :
+   Author :       lihanrui
+   date：          2019/3/6
+-------------------------------------------------
+   Change Activity:
+                   2019/3/6:
+-------------------------------------------------
+"""
+
+
+NOT_FOUND = -1
+LOG_File = "./logs/error.log"
+PROCESS_POOL_SIZE = 8
+
+#虚拟空间页面标志
+XINWANG = u"www.xinnet.com"
+WANWANG = u"cp.hichina.com"
+XIBUSHUMA = u"www.myhostadmin.net"
+MEICHENGKEJI = u"static.websiteonline.cn"
+
+BAIDUCLOUD = u"cloud.baidu.com"
+
+VIRTUAL_SPACE = {
+    XINWANG: u"新网虚拟主机",
+    WANWANG: u"万网虚拟主机",
+    XIBUSHUMA: u"西部数码虚拟主机",
+    MEICHENGKEJI: u"美橙科技虚拟主机",
+
+    BAIDUCLOUD: u"百度云虚拟空间",
+}
+
+#虚拟空间页面目录
+vir_dir = "virtualspace_page_dir"
+
+#CDN页面标识
+BAIDUJIASU = u"su.baidu.com"
+
+CDN = {
+    BAIDUJIASU: u"百度云加速",
+}
+
+#WAF安全标识
+CYD = u"www.365cyd.com"
+
+WAF = {
+    CYD: u"创宇盾",
+
+}
+
+#chinaz站长工具查询接口
+icp_interface = "http://apidata.chinaz.com/CallAPI/Domain?key=%s&domainName=%s"
+whois_interface = "http://apidata.chinaz.com/CallAPI/Whois?key=%s&domainName=%s"
+keys = ["1b63b58928e643c8a7df7ada6236e3b8"]
+
+
+

src/utils/website_info_collect/ipip.py

@@ -0,0 +1,89 @@
+#coding=utf-8
+'''
+Created on 2017年9月13日
+
+@author: bailiangjun
+'''
+
+import struct
+from socket import inet_aton
+import os
+# import settings
+import ipdb
+
+_unpack_V = lambda b: struct.unpack("<L", b)
+_unpack_N = lambda b: struct.unpack(">L", b)
+_unpack_C = lambda b: struct.unpack("B", b)
+class IPX(object):
+    """
+    @attention: 老版本,解析IPV4,已弃用,可删除
+    """
+    binary = ""
+    index = 0
+    offset = 0
+    
+    def __init__(self,fl):
+        path = os.path.abspath(fl)
+        with open(path, "rb") as f:
+            IPX.binary = f.read()
+            IPX.offset, = _unpack_N(IPX.binary[:4])
+            IPX.index = IPX.binary[4:IPX.offset]
+
+    def find(self,ip):
+        """
+        @attention: 解析IP
+        @return: 国,省/州,市,街道,运营商,经度,纬度,时区,时区,行政区划代码,国际区号,顶级域名，未知
+        """
+        index = IPX.index
+        offset = IPX.offset
+        binary = IPX.binary
+        try:
+            nip = inet_aton(ip)
+        except Exception as e:
+            import settings.base_logger as bl
+            import traceback
+            print("%s, ip:%s" % (traceback.format_exc(), ip))
+            return []
+
+        ipdot = ip.split('.')
+        if int(ipdot[0]) < 0 or int(ipdot[0]) > 255 or len(ipdot) != 4:
+            return []
+        tmp_offset = (int(ipdot[0]) * 256 + int(ipdot[1])) * 4
+        start, = _unpack_V(index[tmp_offset:tmp_offset + 4])
+
+        index_offset = index_length = -1
+        max_comp_len = offset - 262144 - 4
+        start = start * 9 + 262144
+
+        while start < max_comp_len:
+            if index[start:start + 4] >= nip:
+                index_offset, = _unpack_V(index[start + 4:start + 7] + chr(0).encode('utf-8'))
+                index_length, = _unpack_C(index[start + 8:start + 9])
+                break
+            start += 9
+
+        if index_offset == 0:
+            return []
+
+        res_offset = offset + index_offset - 262144
+        res = binary[res_offset:res_offset + index_length].decode('utf-8').split("\t")
+        return [item.strip() for item in res]
+
+class IPV6(object):
+    """
+    @attention: 能同时解析IPV4和IPv6
+    """
+
+    def __init__(self,fl):
+        self.db = ipdb.database.Reader(fl)
+        
+    def find(self,ip):
+        try:
+            ip = unicode(ip)
+            info = self.db.find(ip, "CN")
+        except:
+            info = []
+        
+        return info
+#os.path.join("./","ipip.ipdb")
+ip_manager = IPV6("ipip.ipdb")

src/utils/website_info_collect/main.py

@@ -0,0 +1,377 @@
+# -*- coding:utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     icp_check
+   Description :   网站信息查询
+   Author :       lihanrui
+   date：          2019/3/4
+-------------------------------------------------
+   Change Activity:
+                   2019/3/4:
+-------------------------------------------------
+"""
+import requests
+import json
+import socket
+import argparse
+import os
+import time
+import random
+import csv
+
+from multiprocessing import Pool
+socket.setdefaulttimeout(5)
+
+from ipip import ip_manager
+from config import *
+from Logger import Logger
+
+log = Logger.logger
+
+
+def read_file(fpath):
+    if not os.path.exists(fpath):
+        print "%s not exist!" % fpath
+        exit(1)
+    with open(fpath, 'r') as f:
+        return f.readlines()
+
+
+def save_file(filepath, content):
+    try:
+
+        with open(filepath, 'w') as f:
+            f.write(content)
+    except Exception as e:
+        log.debug("%s,%s" % (filepath, e))
+
+
+def http_get(url, headers={}):
+
+    retry = 0
+    headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"})
+
+    while retry < 2:
+        try:
+            response = requests.get(url, headers=headers, timeout=3)
+            response.encoding = 'utf-8'
+            return response.status_code, response.text
+        except Exception as e:
+            retry += 1
+            time.sleep(0.1)
+
+    return 'except', 'except'
+
+
+def check_access_protect(ip, domain):
+
+    ret = dict()
+    if len(ip) == 0:
+        return ret
+
+    access_url = "http://%s/" % ip
+    reps = []
+    for template in ["http://%s/?id=1 or 1=1",
+                     "http://%s/?id=../../etc/passwd",
+                     "http://%s/?id=http://aewrae.aliff.com/1.txt?%%00"]:
+
+        attack_url = template % ip
+        content, code = http_get(attack_url, {"Host": domain})
+        reps.append((content, code))
+
+    ret["access_rep_code"], _ = http_get(access_url, {"Host": domain})
+
+    if ret["access_rep_code"] in [200, 206, 301, 302, 404]:
+        ret["access_rep"] = '是'
+        for code, content in reps:
+            if code != 200 and code != 404:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '是'
+            else:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '否'
+    else:
+        ret["access_rep"] = '否'
+        for code, content in reps:
+            if code != ret["access_rep_code"]:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '是'
+            else:
+                ret["attack_rep_code"] = code
+                ret["safe_device"] = '否'
+
+    return ret
+
+
+def check_virtualspace(ip, domain):
+    ret = dict()
+    if len(ip) == 0:
+        return ret
+
+    url = "http://%s/" % ip
+    code, content = http_get(url)
+    for i in VIRTUAL_SPACE.keys():
+        if content.find(i) == NOT_FOUND:
+            if len(content) != 0:
+                file_name = "%s_%s.html" % (ip, domain)
+                file_path = os.path.join(vir_dir, file_name)
+                save_file(file_path, content)
+        else:
+            ret["virtualspace"] = VIRTUAL_SPACE[i].encode("utf-8")
+
+    for i in CDN.keys():
+        if content.find(i) == NOT_FOUND:
+            continue
+        else:
+            ret["cdn_name"] = CDN[i].encode("utf-8")
+
+    return ret
+
+
+def get_domain_whois(domain):
+
+    def _get(url):
+        try:
+            response = requests.get(url, timeout=3)
+            return response.json()
+        except Exception as e:
+            log.exception("Request whois interface exception:%s" % e)
+        return {}
+
+    ret = dict()
+    domain = domain.strip()
+    if len(domain) == 0:
+        return ret
+
+    url = whois_interface % (random.choice(keys), domain)
+    json_rep = _get(url)
+    if json_rep:
+        status_code = json_rep.get("StateCode", 0)
+        if status_code == 1:
+            data = json_rep.get("Result", "")
+            if not data:
+                log.info(u"%s 暂无Whois信息..." % domain)
+                return ret
+            ret["ExpirationDate"] = data["ExpirationDate"].encode("utf-8")
+        else:
+            log.info(json.dumps(json_rep, ensure_ascii=False))
+    return ret
+
+def get_ip_info(domain):
+
+    def _get_ip(domain):
+        try:
+            myaddr = socket.getaddrinfo(domain, 'http')
+            return myaddr[0][4][0]
+        except Exception as e:
+            log.exception("%s ,%s" % (domain, e))
+            return None
+
+    def _get_ipip_info(ip):
+        ipx_res = ip_manager.find(ip)
+        if len(ipx_res):
+            country = ipx_res[0]
+            province = ipx_res[1]
+            city = ipx_res[2]
+            # 运营商
+            operator = ipx_res[4]
+            return country, province, city, operator
+        else:
+            return ['']*4
+
+    ret = dict()
+    ip = _get_ip(domain)
+    if ip:
+        country, province, city, operator = _get_ipip_info(ip)
+
+        ret["ip"] = ip
+        ret["ip_location"] = ''.join([country, province, city]).encode("utf-8")
+        ret["operator"] = operator.encode("utf-8")
+
+    return ret
+
+
+def get_website_icpinfo(domain):
+
+    domain = domain.strip()
+
+    def _get(url):
+        try:
+            response = requests.get(url, timeout=3)
+            return response.json()
+        except Exception as e:
+            log.exception("Request icp interface exception:%s" % e)
+        return {}
+    ret = dict()
+
+    url = icp_interface % (random.choice(keys), domain)
+
+    json_rep = _get(url)
+
+    if json_rep:
+        status_code = json_rep.get("StateCode", 0)
+        if status_code == 1:
+            data = json_rep.get("Result", "")
+            if not data:
+                log.info(u"%s 暂无备案信息..." % domain)
+                return ret
+            ret["CompanyName"] = data["CompanyName"].encode("utf-8")
+            ret["CompanyType"] = data["CompanyType"].encode("utf-8")
+            ret["MainPage"] = data["MainPage"]
+            ret["SiteLicense"] = data["SiteLicense"].encode("utf-8")
+            ret["SiteName"] = data["SiteName"].encode("utf-8")
+            ret["VerifyTime"] = data["VerifyTime"]
+        else:
+            log.info(json.dumps(json_rep, ensure_ascii=False))
+    return ret
+
+
+def check_expire_date(str_time):
+    try:
+        expire_date = time.strptime(str_time, '%Y-%m-%d %H:%M:%S')
+    except Exception as e:
+        return '不定'
+
+    now = time.time()
+    str_expire = time.mktime(expire_date)
+    if str_expire - now > 0:
+        return '否'
+    else:
+        return '是'
+
+
+
+
+
+
+
+def get_website_info(domain):
+
+    ret_infos = []
+
+    domain = domain.strip()
+
+    #网站IPC信息查询,没有子域名直接返回
+    ipc_info = get_website_icpinfo(domain)
+    mainpage = ipc_info.pop("MainPage", '')
+    if len(mainpage) == 0:
+        info = ['不定']*21
+        info[0] = domain
+        ret_infos.append(info)
+        return ret_infos
+
+    for subdomain in mainpage.split(" "):
+        #网站IP信息查询
+        ip_info = get_ip_info(subdomain)
+        #网站是否可访问、是否存在安全设备
+        access_info = check_access_protect(ip_info.get("ip", ''), subdomain)#(icp_info["MainPage"])
+        #网站是否在虚拟空间
+        virtualspace = check_virtualspace(ip_info.get("ip", ''), subdomain)
+        #域名whois信息(过期时间)
+        whois_info = get_domain_whois(subdomain)
+
+        ret_infos.append(
+            [domain.encode('utf-8'), subdomain.encode('utf-8'),
+             whois_info.get("ExpirationDate", '不定').strip(), check_expire_date(whois_info.get("ExpirationDate", '不定').strip()),
+             ipc_info.get("SiteName", '不定'), access_info.get("access_rep", '否'),
+             ip_info.get("ip", '不定'), ip_info.get("ip_location", '不定'), ip_info.get("operator", '不定'),
+             ipc_info.get("CompanyType", '不定'), ipc_info.get("CompanyName", '不定'),
+
+             '是' if ipc_info.get("SiteLicense", '') != "" else '否', ipc_info.get("SiteLicense", '不定'),
+             '',
+             '是' if virtualspace.get("virtualspace", '') != "" else '不定', virtualspace.get("virtualspace", '不定'),
+              access_info.get("safe_device", '不定'),
+             '是' if virtualspace.get("cdn_name", '') != "" else '不定', virtualspace.get("cdn_name", '不定'),
+
+             time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), '']
+        )
+    return ret_infos
+
+
+def output_csv(infos):
+
+    localtime = time.localtime()
+    strtime = time.strftime("%Y%m%d%H%M%S", localtime)
+    fname = "outfile_%s.csv" % strtime
+
+    headers = ["根域名", "子域名枚举",
+               "域名到期时间","域名是否到期",
+               "网站名称", "是否正常访问",
+               "所属IP", "地理位置", "运营商",
+               "主办单位性质", "主办单位名称",
+
+               "是否备案", "ICP备案号",
+               "备案时间",
+               "是否虚拟空间接入", "虚拟空间服务商",
+               "是否有安全防护",
+               "是否有CDN","CDN厂商",
+
+               "检查时间", "备注"]
+    with open(fname, 'wb') as f:
+        writer = csv.writer(f)
+        writer.writerow(headers)
+        for info in infos:
+            try:
+                writer.writerow(info)
+            except Exception as e:
+                log.exception("%s" % e)
+
+
+def init():
+
+    if not os.path.exists(os.path.dirname(LOG_File)):
+        os.mkdir(os.path.dirname(LOG_File))
+
+    if not os.path.exists(vir_dir):
+        os.mkdir(vir_dir)
+
+    Logger.add_file_handler(LOG_File)
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(prog="icp_info_check", usage="%(prog)s [options] usage", description="Website Information (Incloud: ICP Record Information) Query Tool")
+    parser.add_argument("-n", "--name", help="The root domain name you want to find")
+    parser.add_argument("-f", "--filepath", help="The root domain name list path")
+    parser.add_argument('-v', '--version', action='version', version='version 0.0.1')
+
+    args = parser.parse_args()
+
+    init()
+    if args.filepath:
+
+        filepath = args.filepath
+        lines = read_file(filepath)
+        infos = list()
+        p = Pool(PROCESS_POOL_SIZE)
+        l_result = p.map(get_website_info, lines)
+        for r in l_result:
+            infos.extend(r)
+        if len(infos) != 0:
+            output_csv(infos)
+        else:
+            print "No website parsing success!"
+
+    elif args.name:
+        headers = ["根域名", "子域名枚举",
+                   "域名到期时间","域名是否到期",
+                   "网站名称", "是否正常访问",
+                   "所属IP", "地理位置", "运营商",
+                   "主办单位性质", "主办单位名称",
+                   "是否备案", "ICP备案号",
+                   "备案时间",
+                   "是否虚拟空间接入", "虚拟空间服务商",
+                   "是否有安全防护",
+                   "是否有CDN", "CDN厂商",
+                   "检查时间", "备注"]
+        domain = args.name.strip()
+        info = get_website_info(domain)
+        format_info = dict(zip(headers, info[0]))
+        print json.dumps(format_info, ensure_ascii=False, indent=4).decode("utf-8").encode("gbk")
+
+
+
+
+
+
+
+

src/utils/website_info_collect/requirements.txt

@@ -0,0 +1,3 @@
+requests==2.9.1
+ipip-ipdb==1.2.1
+gevent==1.1.1

src/utils/website_info_collect/test.py

@@ -0,0 +1,4 @@
+#coding:utf-8
+import main
+
+print main.get_website_icpinfo("www.baidusdfs.com")

src/wsgi.py

@@ -0,0 +1,22 @@
+# encoding=utf8
+"""
+WSGI config for cloud_defense project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
+"""
+
+import sys
+reload(sys)
+sys.setdefaultencoding("utf-8")
+
+import os
+
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+
+application = get_wsgi_application()

src/xeshop/control_contacts.py

@@ -0,0 +1,82 @@
+# -*-coding:utf-8-*-
+import datetime
+
+import common.error_info as ce
+import common.models as cm
+import common.common_functions as ccf
+import utils.constant as const
+
+
+def add_contacts(request):
+    """
+    """
+    uid = request.user.id
+    kwargs = request.json
+    need_params = ["name","phone","email"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    need_params.append("remark")
+    cvals = ccf.get_need_params(*need_params,**kwargs)
+    cvals.update({"user_id":uid})
+    cm.Contacts.objects.create(**cvals)
+
+    
+def update_contacts(request):
+    """
+    """
+    uid = request.user.id
+    kwargs = request.json
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    id = kwargs.get("id")
+    uvals = {}
+    if "name" in kwargs:
+        uvals.update({"name":kwargs.get("name")})
+    if "email" in kwargs:
+        uvals.update({"email":kwargs.get("email")})
+    if "phone" in kwargs:
+        uvals.update({"phone":kwargs.get("phone")})
+    if "remark" in kwargs:
+        uvals.update({"remark":kwargs.get("remark")})
+    cm.Contacts.objects.filter(id=id).update(**uvals)
+
+
+def delete_contacts(request):
+    """删除批量用,隔开
+    """
+    kwargs = request.json
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    id = kwargs.get("id")
+    ids = str(id).split(",")
+    cm.Contacts.objects.filter(id__in=ids).delete()
+
+
+def get_contacts_info(*ids):
+    """获取联系人详情
+    """
+    if not ids:
+        raise ce.TipException(u"缺少参数id!")
+    if not cm.Contacts.objects.filter(id__in=ids).exists():
+        raise ce.TipException(u"资源不存在!")
+    qdata = cm.Contacts.objects.filter(id__in=ids).values()
+    return list(qdata)
+
+
+def get_contacts_list(request):
+    """
+    """
+    uid = request.user.id
+    filter_info = request.json
+    qset = cm.Contacts.objects.filter(user__id=uid)
+    if filter_info.get("name"):
+        qset = qset.filter(name__icontains=filter_info.get("name"))
+    qdata = list(qset.values())
+
+    return qset.count(),qdata
+

src/xeshop/control_notice.py

@@ -0,0 +1,120 @@
+# -*-coding:utf-8-*-
+import datetime
+
+import common.error_info as ce
+import common.models as cm
+import common.common_functions as ccf
+import common.common_control as ccc
+import utils.constant as const
+
+
+def add_sysnotice(request):
+    """
+    """
+    uid = request.user.id
+    uname = request.user.name
+    kwargs = request.json
+    need_params = ["title","content"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    cvals = ccf.get_need_params(*need_params,**kwargs)
+    cvals.update({"cid":uid})
+    cvals.update({"cperson":uname})
+    cm.SysNotice.objects.create(**cvals)
+
+
+def update_sysnotice(id,**kwargs):
+    """
+    """
+    uvals = {}
+    org_viewer = cm.SysNotice.objects.filter(id=id).first().viewer
+    org_viewer = org_viewer if org_viewer else ""
+    org_viewer = org_viewer.split(",")
+    if "viewer" in kwargs:
+        viewer = kwargs.get("viewer")
+        org_viewer.append(str(viewer))
+        viewer = ",".join(list(set(org_viewer)))
+        uvals.update({"viewer":viewer})
+    cm.SysNotice.objects.filter(id=id).update(**uvals)
+
+    
+def delete_sysnotice(request):
+    """删除批量用,隔开
+    """
+    kwargs = request.json
+    need_params = ["id"]
+    mse = ccf.check_params(*need_params,**kwargs)
+    if mse:
+        raise ce.TipException(mse)
+    id = kwargs.get("id")
+    ids = str(id).split(",")
+    cm.SysNotice.objects.filter(id__in=ids).delete()
+
+
+def get_sysnotice_info(*ids):
+    """获取系统通知详情
+    """
+    if not ids:
+        raise ce.TipException(u"缺少参数id!")
+    if not cm.SysNotice.objects.filter(id__in=ids).exists():
+        raise ce.TipException(u"资源不存在!")
+    qdata = cm.SysNotice.objects.filter(id__in=ids).values()
+    return list(qdata)
+
+
+def get_sysnotice_list(request):
+    """
+    """
+    uid = request.user.id
+    filter_info = request.json
+    qset = cm.SysNotice.objects.filter(cid=uid)
+    if filter_info.get("title"):
+        qset = qset.filter(title=filter_info.get("title"))
+    # if filter_info.get("stime"):
+    #     qset = qset.filter(ctime__gte=filter_info.get("stime"))
+    # if filter_info.get("etime"):
+    #     qset = qset.filter(ctime__lte=filter_info.get("etime"))
+    if filter_info.get("ctime", ""):
+        import web_site.tools as wst
+        query = wst.datetime_filter("ctime", {}, filter_info["ctime"])
+        qset = qset.filter(**query)
+    if filter_info.get("new"):
+        vuid = str(uid)
+        qset = qset.exclude(viewer__contains=vuid)
+    qdata = list(qset.values())
+
+    return qset.count(),qdata
+
+
+def get_warning_notice_list(request):
+    filter_info = request.json
+    qset = cm.WarningNotice.objects.filter(is_suc=1)
+    if filter_info.get("warning_type"):
+        qset = qset.filter(warning_type=filter_info.get("warning_type"))
+    if filter_info.get("notice_type"):
+        qset = qset.filter(notice_type=filter_info.get("notice_type"))
+    # if filter_info.get("stime"):
+    #     qset = qset.filter(ctime__gte=filter_info.get("stime"))
+    # if filter_info.get("etime"):
+    #     qset = qset.filter(ctime__lte=filter_info.get("etime"))
+    if filter_info.get("ctime", ""):
+        import web_site.tools as wst
+        query = wst.datetime_filter("ctime", {}, filter_info["ctime"])
+        qset = qset.filter(**query)
+    count = qset.count()
+    page = filter_info.get("page")
+    page_size = filter_info.get("page_size")
+    if page and page_size:
+        count,qset = ccc.get_page_qset(qset,page,page_size)
+    qdata = list(qset.values())
+
+    return count,qdata
+
+
+def get_warning_notice(*ids):
+    """
+    """
+    qset = cm.WarningNotice.objects.filter(id__in=ids)
+    qdata = qset.values()
+    return qdata

src/xeshop/templates/apidoc.html

@@ -0,0 +1,169 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<title></title>
+	<style type="text/css">
+		.container{
+			width: 100%;
+		}
+		.left{
+			position:fixed;
+			width: 20%;
+			float: left;
+			border-right:1px solid #ccc;
+			height: 100%;
+            overflow:scroll;
+		}
+		.right{
+			width:79%;
+			float: right;
+			height: 100%;
+			padding-top:5px;
+		}
+		.clear{
+			clear: both;
+		}
+		label{
+			display: block;
+			margin:10px 0px;
+		}
+		a{
+			text-decoration: none;
+		}
+		.delapi{
+			float: right;
+			margin-right: 10px;
+			cursor: pointer;
+		}
+	</style>
+</head>
+<body>
+	<div class="container">
+		<div class="left">
+			<ol>
+				{% for api in apis %}
+				<li draggable="true" id="{{api.id}}"><a href="#{{api.url}}{{api.method}}" >{{api.title}}</a>
+					{% if isroot %}
+					<button class="delapi">x</button>
+					{% endif %}
+				</li>
+				{% endfor %}
+			</ol>
+		</div>
+		<div class="right">
+			{% for api in apis %}
+				<div id="{{api.url}}{{api.method}}">
+					<div style="background: #ccc;margin-top:20px;"><b>{{api.url}} &nbsp;&nbsp;&nbsp;&nbsp;{{api.method}}</b></div>
+					<label>参数:</label>
+					<div style="margin-left:25px;">
+						{% if api.params %}
+						<textarea cols="80" rows="20">{{api.params}}</textarea>
+							{% if api.request_note %}
+								<label>说明:</label>
+								<div>{{api.request_note|safe}}</div>
+							{% endif %}
+						{% else %}
+						无
+						{% endif %}
+					</div>
+					<label>响应:</label>
+					<div style="margin-left:25px;">
+						{% if api.response %}
+						<textarea class="response" cols="80" rows="20">{{api.response}}</textarea>
+							{% if api.response_note %}
+								<label>说明:</label>
+								<div>{{api.response_note|safe}}</div>
+							{% endif %}
+						{% else %}
+						无
+						{% endif %}
+					</div>
+				</div>
+			{% endfor %}
+		</div>
+		<div class="clear"></div>
+	</div>
+</body>
+<script
+  src="https://code.jquery.com/jquery-1.12.4.min.js"
+  integrity="sha256-ZosEbRLbNQzLpnKIkEdrPv7lOy9C27hHQ+Xp8a4MxAQ="
+  crossorigin="anonymous"></script>
+<script type="text/javascript">
+    function insertAfter(newEl, targetEl)
+        {
+            var parentEl = targetEl.parentNode;
+            if(parentEl.lastChild == targetEl)
+                {
+                    parentEl.appendChild(newEl);
+                }else
+                {
+                    parentEl.insertBefore(newEl,targetEl.nextSibling);
+                }            
+        }
+    //
+    var res = document.getElementsByClassName("response")
+    for(var i=0;i<res.length;i++){
+        res[i].innerHTML = JSON.stringify(JSON.parse(res[i].value),undefined,4) 
+    }
+
+	//
+	let parent = document.getElementsByTagName("ol")[0]
+	let div = document.getElementsByTagName("li")
+    let container = null
+    for(let i = 0; i < div.length; i++) {
+        div[i].ondragstart = function() {
+            // 当拖动其中一个元素时，this的指向便是你所拖动的元素，将它存在container
+            container = this
+        }
+        // 默认的当你dragover的时候会阻止你做drop的操作，所以需要取消这个默认
+        div[i].ondragover = function() {
+            event.preventDefault()
+        }
+        // 当拖动结束时，给拖动div所在位置下面的div做drop事件，注意drop时this的指向发生改变
+        div[i].ondrop = function()
+        {
+            if(container != null && container != this)
+            {
+                insertAfter(container,this)
+                //let temp = document.createElement("li")
+                //parent.replaceChild(temp,this)
+                //parent.replaceChild(this,container)
+                //parent.replaceChild(container,temp)
+				//
+				let dataobj = []
+				for(var i=0;i<div.length;i++){
+					console.log(div[i].id)
+					let id = div[i].id
+					dataobj.push({"id":id,"order":i})
+				}
+				$(function(){
+					$.ajax({
+						type:"POST",
+						url:"/api/doc/",
+						contentType:"application/json",
+						data:JSON.stringify({"ids":dataobj}),
+						success:function(){
+							window.location.reload();
+						}
+					})
+				})
+            }
+        }
+    }
+$(function(){
+	$(".delapi").click(function(e){
+		let id = e.target.parentNode.id;
+		$.ajax({
+			type:"DELETE",
+			url:"/api/doc/",
+			contentType:"application/json",
+			data:JSON.stringify({"id":id}),
+			success:function(){
+				window.location.reload();
+			}
+		})
+	});
+});
+
+</script>
+</html>

src/xeshop/urls_backstage.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+'''
+'''
+from django.conf.urls import url
+import views_contacts,views_notice
+
+urlpatterns = [
+    url(r'^contacts$', views_contacts.ContactsView.as_view()),
+    url(r'^contacts/list$', views_contacts.ContactsListView.as_view()),
+    url(r'^notice/sys$', views_notice.SysNoticeView.as_view()),
+    url(r'^notice/sys/list$', views_notice.SysNoticeListView.as_view()),
+    url(r'^notice/warning$', views_notice.WarningNoticeView.as_view()),
+    url(r'^notice/warning/list$', views_notice.WarningNoticeListView.as_view()),
+]
+

src/xeshop/views.py

@@ -0,0 +1,81 @@
+# coding=utf-8
+import re 
+import os
+import json
+import uuid
+import random
+from importlib import import_module
+
+from django import http
+from django.contrib import auth
+from django.core.cache import cache
+from django.shortcuts import render
+
+from common import core_views as cv
+import common.models as cm
+
+class DocView(cv.BaseView):
+    def get(self,request):
+        """
+        """
+        modules = list(cm.OperationLogConfig.objects.order_by("order").values("id","op_view","op_action_flag","op_url","op_response"))
+        apis = []
+        for mo in modules:
+            docstr = None
+            op_url = mo["op_url"]
+            response = mo["op_response"]
+            cls = mo["op_view"].split(".")[-1]
+            mod = import_module(".".join(mo["op_view"].split(".")[:-1]))
+            try:
+                cls = getattr(mod,cls)
+                op_action_flag = mo["op_action_flag"]
+                if op_action_flag == "GET":
+                    docstr = cls.get.__doc__
+                if op_action_flag == "POST":
+                    docstr = cls.post.__doc__
+                if op_action_flag == "PUT":
+                    docstr = cls.put.__doc__
+                if op_action_flag == "DELETE":
+                    docstr = cls.delete.__doc__
+                if docstr:
+                    title = "".join(re.findall(r'#.*',docstr)).lstrip("#")
+                    params = "\n".join([x.lstrip("@") for x in re.findall(r'@.*',docstr)]).strip()
+                    request_note = "<br>".join([x.lstrip("<") for x in re.findall(r'<.*',docstr)]).strip()
+                    response_note = "<br>".join([x.lstrip(">") for x in re.findall(r'>.*',docstr)]).strip()
+                    apis.append({"id":mo["id"],"url":op_url,"title":title,"method":op_action_flag,
+                                 "params":params,"response":response,"response_note":response_note,
+                                "request_note":request_note
+                                })
+            except Exception as e:
+                cv.tracefail()
+        isroot = request.GET.get("root",0)
+        return render(request,"apidoc.html",{"apis":apis,"isroot":isroot})
+
+    def post(self,request):
+        """
+        """
+        qdata = request.json
+        ids = qdata.get("ids")
+        for dct in ids:
+            id = dct["id"]
+            order = dct["order"]
+            cm.OperationLogConfig.objects.filter(id=id).update(order=order)
+        return cv.to_suc()
+
+
+    def delete(self,request):
+        """
+        """
+        qdata = request.json
+        id = qdata.get("id")
+        cm.OperationLogConfig.objects.filter(id=id).delete()
+        return cv.to_suc()
+
+
+class HookView(cv.BaseView):
+    def post(self,request):
+        """
+        """
+        cmdstr = "cd /home/work/365gcd_wsm_backend/src & git pull & monit restart wsm"
+        os.system(cmdstr)
+        return cv.to_suc()

src/xeshop/views_contacts.py

@@ -0,0 +1,84 @@
+#-*-coding:utf-8-*-
+import json
+import uuid
+from django.contrib import auth
+from django.core.cache import cache
+from django.conf import settings
+
+from common import core_views as cv
+from common.models import UserInfo
+import common.common_functions as ccf
+import common.error_info as ce
+import wsm.control_contacts as cct
+
+
+class ContactsView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取联系人详情
+        @id:1,联系人id
+        """
+        try:
+            kwargs = request.json
+            need_params = ["id"]
+            mse = ccf.check_params(*need_params,**kwargs)
+            if mse:
+                raise ce.TipException(mse)
+            ids = str(kwargs.get("id")).split(",")
+            res = cct.get_contacts_info(*ids)
+            res = res[0] if res else {}
+            return cv.to_suc(res)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def post(self,request):
+        """
+        #新增联系人
+        @name:"名称"
+        @email:"496498291@qq.com"
+        @phone:"496498291@qq.com"
+        @remark:"备注"
+        """
+        try:
+            cct.add_contacts(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def put(self,request):
+        """
+        #修改联系人
+        @id:1,其他参数同新增新增
+        """
+        try:
+            cct.update_contacts(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+    def delete(self,request):
+        """
+        #删除联系人
+        @id:1
+        """
+        try:
+            cct.delete_contacts(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+
+class ContactsListView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取联系人列表
+        @name:"名称",联系人名称筛选
+        """
+        try:
+            count,res = cct.get_contacts_list(request)
+            return cv.to_suc({"total":count,"list":res})
+        except Exception as e:
+            return cv.to_fail(e)

src/xeshop/views_notice.py

@@ -0,0 +1,110 @@
+#-*-coding:utf-8-*-
+import json
+import uuid
+from django.contrib import auth
+from django.core.cache import cache
+from django.conf import settings
+
+from common import core_views as cv
+from common.models import UserInfo
+import common.common_functions as ccf
+import common.error_info as ce
+import wsm.control_notice as cnt
+
+
+class SysNoticeView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取系统通知详情
+        @id:1,id
+        """
+        try:
+            kwargs = request.json
+            need_params = ["id"]
+            mse = ccf.check_params(*need_params,**kwargs)
+            if mse:
+                raise ce.TipException(mse)
+            ids = str(kwargs.get("id")).split(",")
+            res = cnt.get_sysnotice_info(*ids)
+            res = res[0] if res else {}
+            for id in ids:
+                cnt.update_sysnotice(id,**{"viewer":request.user.id})
+            return cv.to_suc(res)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def post(self,request):
+        """
+        #新增系统公告
+        @title:"标题"
+        @content:"公告内容"
+        """
+        try:
+            cnt.add_sysnotice(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+    def delete(self,request):
+        """
+        #删除系统公告
+        @id:1
+        """
+        try:
+            cnt.delete_sysnotice(request)
+            return cv.to_suc()
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+
+class SysNoticeListView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取系统公告列表
+        @title:"名称",联系人名称筛选
+        @ctime:"2020-03-04 00:00:00,2020-03-05 00:00:00" 时间范围
+        """
+        try:
+            count,res = cnt.get_sysnotice_list(request)
+            return cv.to_suc({"total":count,"list":res})
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WarningNoticeView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取告警通知记录详情
+        @id:1,告警记录id
+        """
+        kwargs = request.json
+        need_params = ["id"]
+        mse = ccf.check_params(*need_params,**kwargs)
+        if mse:
+            raise ce.TipException(mse)
+        try:
+            id = kwargs.get("id")
+            wnotice = cnt.get_warning_notice(*[id])
+            wnotice = wnotice[0] if wnotice else {}
+            return cv.to_suc(wnotice)
+        except Exception as e:
+            return cv.to_fail(e)
+
+
+class WarningNoticeListView(cv.AuthView):
+    def get(self, request):
+        """
+        #获取告警通知记录列表
+        @warning_type:"bug",告警类型
+        @notice_type:"email",告警方式(email,msg)
+        @ctime:"2020-03-04 00:00:00,2020-03-05 00:00:00" 时间范围
+        """
+        try:
+            count,res = cnt.get_warning_notice_list(request)
+            return cv.to_suc({"total":count,"list":res})
+        except Exception as e:
+            return cv.to_fail(e)
+