class UploadPcapForm(forms.Form):
    pcap_file = forms.FileField(label="pcap file") class Handler:
    def __init__(self):
        self.file_path = None
        self.file_name = None
        self.stream = None
        self.data = None
        self.log = Logger("File Handler", "DEBUG")

    def create_dir(self):
        now = datetime.datetime.now()
        self.log.message("Now is: %s:" % now)
        directory_name = now.strftime("%d-%m-%y")
        self.log.message("Directory name: %s:" % directory_name)
        directory_path = "/".join([settings.PROJECT_ROOT, "uploads", directory_name])
        self.log.message("Directory path: %s" % directory_path)
        if not os.path.exists(directory_path):
            os.mkdir(directory_path)
            self.log.message("Directory created")
        # we need to create another directory also for each upload
        new_dir = generate_name_from_timestame()
        new_dir_path = "/".join([directory_path, new_dir])
        if not os.path.exists(new_dir_path):
            os.mkdir(new_dir_path)
            self.log.message("Directory created")
        self.upload_dir = new_dir_path

    def save_file(self, f):
        self.file_name = f.name
        self.file_path = "/".join([self.upload_dir, self.file_name])
        destination = open(self.file_path, 'wb+')
        for chunk in f.chunks():
            destination.write(chunk)
        destination.close()

    def search(self, file_path, strings=None):
        try:
            self.stream = FileInputStream(unicodeFilename(file_path), real_filename=file_path)
        except NullStreamError:
            return False
        patterns = PatternMatching()
        for s in strings:
            patterns.addString(s)

        start = 0
        end = self.stream.size
        self.data = self.stream.readBytes(start, end//8)
        return patterns.search(self.data)

    def reset_data(self):
        self.data = None class Handler:

    def __init__(self, debug_mode="DEBUG"):
        self._logger = Logger(log_name="Pcap Handler", log_mode=debug_mode)
        self._logger.message("Pcap Handler initialized")
        self._pcap = None
        self._filter_type = None
        self._file_pointer = None

    def open_file(self, pcap_file, mode="rb"):
        try:
            self._file_pointer = file(pcap_file, mode)
            self._logger.set_log_level("DEBUG")
            self._logger.message(("%s is opened at %s mode") % (pcap_file, mode))
        except:
            self._logger.set_log_level("ERROR")
            self._logger.message("Error at opening pcap file")

    def open_pcap(self, mode="r"):
        if mode == "r":
            self._pcap = dpkt.pcap.Reader(self._file_pointer)
            self._logger.set_log_level("DEBUG")
            self._logger.message("pcap reader is created")
        if mode == "w":
            self._pcap = dpkt.pcap.Writer(self._file_pointer)

    def write_pcap(self, buf, ts):
        self._pcap.writepkt(buf, ts)

    def close_file(self):
        self._file_pointer.close()

    def set_filter_type(self, t):
        self._filter_type = t
        self._logger.set_log_level("DEBUG")
        self._logger.message(("Filter type is set %s") % (t))

    def get_filter_type(self):
        return self._filter_type

    def get_pcap(self):
        return self._pcap

    def get_eth(self, buf):
        eth = dpkt.ethernet.Ethernet(buf)
        if eth.type == dpkt.ethernet.ETH_TYPE_IP:
            return eth
        else:
            self._logger.set_log_level("ERROR")
            self._logger.message("No Eth is returned")
            return False

    def get_ip(self, eth):
        ip = eth.data
        if ip.p == dpkt.ip.IP_PROTO_TCP:
            return ip
        else:
            self._logger.set_log_level("ERROR")
            self._logger.message("No IP is returned")
            return False

    def get_tcp(self, ip):
        tcp = ip.data
        #self._logger.message(("TCP is returned %s") % (tcp))
        return tcp

    def get_udp(self, ip):
        udp = ip.data
        return udp

    def get_reader(self):
        return self._pcap class Handler:

    def __init__(self, handler, debug_mode="DEBUG"):
        self.pcap = handler.get_pcap()
        self.pcap_handler = handler
        self._logger = Logger(log_name="Flow Handler", log_mode=debug_mode)
        self._logger.message("Flow Handler initialized")

    def get_tcp_flows(self, filter_type="TCP"):
        flow = dict()
        flow_id = 0
        flow_num = 0
        direction = dict() # 1 is one, 2 is bidirectional, keep the flow numbers as indexes
        index = dict()
        self.pcap_handler.set_filter_type(filter_type)
        for ts, buf in self.pcap:
            eth = self.pcap_handler.get_eth(buf)
            if eth:
                ip = self.pcap_handler.get_ip(eth)
            else:
                continue
            #src_ip = self.ip.src
            #dst_ip = self.ip.dst
            # for human readable ip
            # from socket import inet_ntoa
            # inet_ntoa(dst_ip)
            if self.pcap_handler.get_filter_type() == "TCP":
                if not ip:
                    continue
                tcp = self.pcap_handler.get_tcp(ip)
                forward_index = (ip.src, tcp.sport, ip.dst, tcp.dport)
                backward_index = (ip.dst, tcp.dport, ip.src, tcp.sport)
                if index.has_key(forward_index):
                    flow_num = index[forward_index]
                elif index.has_key(backward_index):
                    flow_num = index[backward_index]
                    direction[flow_num] = 2
                else:
                    index[forward_index] = flow_id
                    flow_num = flow_id
                    direction[flow_num] = 1

                if flow.has_key(flow_num):
                    flow[flow_num].append((buf,ts))
                else:
                    flow[flow_num] = [(buf, ts)]
            flow_id += 1
        return flow, direction

    def save_flow(self, flow, pcap_handler, save_path=""):
        random_key = generate_random_name(10)
        files = dict()
        for key, values in flow.iteritems():
            file_name = ".".join([random_key, str(key), "pcap"])
            full_file_path = "/".join([save_path, file_name])
            if files.has_key(save_path):
                files[save_path].append(file_name)
            else:
                files[save_path] = [file_name]
            pcap_handler.open_file(full_file_path, "w")
            pcap_handler.open_pcap("w")
            for value in values:
                pcap_handler.write_pcap(value[0], value[1])
            pcap_handler.close_file()
        return files class Handler(object):
    def __init__(self):
        super(Handler, self).__init__()
        self.timestamp = None
        self.proto = None
        self.src_ip = None
        self.dst_ip = None
        self.sport = None
        self.dport = None
        self.ident = None
        self.length = None
        self.data = None
        self.log = Logger("TCP Protocol Handler", "DEBUG")
        self.log.message("TCP protocol handler called")

    def read_tcp(self, ts, buf):
        eth = self.get_eth(buf)
        if not eth:
            return False
        ip = self.get_ip(eth)
        if not ip:
            return False
        self.timestamp = datetime.datetime.fromtimestamp(float(ts))
        tcp = self.get_tcp(ip)
        return tcp

    def get_eth(self, buf):
        eth = dpkt.ethernet.Ethernet(buf)
        if eth.type != dpkt.ethernet.ETH_TYPE_IP:
            return False
        else:
            return eth

    def get_ip(self, eth):
        ip = eth.data
        self.length = ip.len
        if ip.p != dpkt.ip.IP_PROTO_TCP:
            return False
        else:
            self.proto = ip.p
            self.src_ip = '.'.join(str(ord(c)) for c in ip.src)
            self.dst_ip = '.'.join(str(ord(c)) for c in ip.dst)
            return ip

    def get_tcp(self, ip):
        tcp = ip.data
        self.ident = ip.id
        self.sport = tcp.sport
        self.dport = tcp.dport
        self.data = tcp.data
        return tcp class Handler(object):
    def __init__(self):
        super(Handler, self).__init__()
        self.timestamp = None
        self.proto = None
        self.src_ip = None
        self.dst_ip = None
        self.sport = None
        self.dport = None
        self.ident = None
        self.length = None
        self.data = None
        self.log = Logger("TCP Protocol Handler", "DEBUG")
        self.log.message("TCP protocol handler called")

    def read_udp(self, ts, buf):
        eth = self.get_eth(buf)
        if not eth:
            return False
        ip = self.get_ip(eth)
        if not ip:
            return False
        self.timestamp = datetime.datetime.fromtimestamp(float(ts))
        udp = self.get_udp(ip)
        return udp

    def get_eth(self, buf):
        eth = dpkt.ethernet.Ethernet(buf)
        if eth.type != dpkt.ethernet.ETH_TYPE_IP:
            return False
        else:
            return eth

    def get_ip(self, eth):
        ip = eth.data
        self.length = ip.len
        if ip.p != dpkt.ip.IP_PROTO_UDP:
            return False
        else:
            self.proto = ip.p
            self.src_ip = '.'.join(str(ord(c)) for c in ip.src)
            self.dst_ip = '.'.join(str(ord(c)) for c in ip.dst)
            return ip

    def get_udp(self, ip):
        udp = ip.data
        self.ident = ip.id
        self.sport = udp.sport
        self.dport = udp.dport
        self.data = udp.data
        return udp class UserJSonFile(models.Model):
    user_id = models.CharField(max_length=100)
    json_type = models.CharField(max_length=10) # possible value is summary for the summary view
    json_file_name = models.CharField(max_length=100) # save the name of the already created file name on disk class Handler:
    def __init__(self):
        self._file_name = None

    def set_file(self, file_name):
        self._file_name = file_name

    def get_hash(self, file_name):
        self._file_name = file_name
        md5 = hashlib.md5()
        with open(self._file_name, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), b''):
                md5.update(chunk)
        return md5.hexdigest() class Flow(models.Model):
    user_id = models.CharField(max_length=100)
    hash_value = models.CharField(max_length=50)
    file_name = models.CharField(max_length=50)
    upload_time = models.DateTimeField()
    file_type = models.CharField(max_length=150)
    file_size = models.IntegerField()
    path = models.FilePathField()
    pcaps = ListField(EmbeddedModelField('Pcap', null=True, blank=True))
    details = ListField(EmbeddedModelField('FlowDetails', null=True, blank=True))

    def __unicode__(self):
        return u'%s/%s' % (self.path, self.file_name)

    def get_upload_path(self):
        hash_dir = os.path.basename(self.path)
        root = os.path.basename(os.path.dirname(self.path))
        return os.path.join(root, hash_dir) class Pcap(models.Model):
    hash_value = models.CharField(max_length=100)
    file_name = models.FileField(upload_to="uploads", null=True, blank=True)
    path = models.FilePathField()
    packets = ListField(EmbeddedModelField('PacketDetails', null=True, blank=True))

    def __unicode__(self):
        return u'%s/%s' % (self.path, self.file_name)

    def get_upload_path(self):
        hash_dir = os.path.basename(self.path)
        root = os.path.basename(os.path.dirname(self.path))
        return os.path.join(root, hash_dir) class PacketDetails(models.Model):
    #datetime.datetime.fromtimestamp(float("1286715787.71")).strftime('%Y-%m-%d %H:%M:%S')
    ident = models.IntegerField()
    flow_hash = models.CharField(max_length=50)
    timestamp = models.DateTimeField()
    length = models.IntegerField()
    protocol = models.IntegerField()
    src_ip = models.IPAddressField()
    dst_ip = models.IPAddressField()
    sport = models.IntegerField()
    dport = models.IntegerField()
    data = models.TextField(null=True, blank=True)

    def __unicode__(self):
        return u'(%s, %s, %s, %s, %s)' % (self.protocol, self.src_ip, self.sport, self.dst_ip, self.dport)

    objects = MongoDBManager() class FlowDetails(models.Model):
    parent_hash_value = models.CharField(max_length=50)
    user_id = models.CharField(max_length=100)
    src_ip = models.IPAddressField()
    dst_ip = models.IPAddressField()
    sport = models.IntegerField()
    dport = models.IntegerField()
    protocol = models.CharField(max_length=10)
    timestamp = models.DateTimeField()

    objects = MongoDBManager() class HTTPDetails(models.Model):
    # request or response
    http_type = models.CharField(max_length=10)
    # request fields
    method = models.CharField(max_length=5, null=True, blank=True)
    uri = models.URLField(null=True, blank=True)
    headers = models.TextField(null=True, blank=True)
    version = models.FloatField(null=True, blank=True)
    # request part ends
    # response fields
    # header and version is here also
    reason = models.CharField(max_length="5", null=True, blank=True)
    status = models.IntegerField(null=True, blank=True)
    # i might need body
    body = models.TextField(null=True, blank=True)
    content_type = models.CharField(max_length=25, null=True, blank=True)
    content_encoding = models.CharField(max_length=25, null=True, blank=True)
    # response ends
    # i might need files also
    files = ListField(null=True, blank=True)
    file_path = models.CharField(max_length=200, null=True, blank=True)
    flow_details = EmbeddedModelField('FlowDetails', null=True, blank=True)

    #for raw_qeuries, filtering according to flow_details will be possible
    objects = MongoDBManager() class DNSRequest(models.Model):
    type = models.IntegerField()
    human_readable_type = models.CharField(max_length=50)
    value = models.CharField(max_length=50, null=True, blank=True)
    flow_details = EmbeddedModelField('FlowDetails', null=True, blank=True)

    objects = MongoDBManager() class DNSResponse(models.Model):
    type = models.IntegerField()
    human_readable_type = models.CharField(max_length=50)
    value = ListField(null=True, blank=True)
    flow_details = EmbeddedModelField('FlowDetails', null=True, blank=True)

    objects = MongoDBManager() class SMTPDetails(models.Model):
    login_data = ListField(null=True, blank=True)
    msg_from = models.CharField(max_length=100, null=True, blank=True)
    rcpt_to = models.CharField(max_length=100, null=True, blank=True)
    raw = models.TextField(null=True, blank=True)
    msgdata = models.TextField(null=True, blank=True)
    attachment_path = ListField(null=True, blank=True)
    flow_details = EmbeddedModelField('FlowDetails', null=True, blank=True)

    objects = MongoDBManager()

    def get_path_dict(self):
        #/home/oguz/git/ovizart/ovizart/uploads/16-06-12/a6a6defb7253043a55281d01aa66538a/smtp-messages/1/part-001.ksh
        result = []
        for path in self.attachment_path:
            tmp = dict()
            r = path.split("uploads")
            file_name = os.path.basename(r[1])
            tmp['file_name'] = file_name
            tmp['path'] = r[1]
            result.append(tmp)

        return result def translate_time(value, leftMin=MIN_TS, leftMax=MAX_TS, rightMin=1, rightMax=100):
    # Figure out how 'wide' each range is
    leftSpan = leftMax - leftMin
    rightSpan = rightMax - rightMin

    # Convert the left range into a 0-1 range (float)
    valueScaled = float(value - leftMin) / float(leftSpan)

    # Convert the range into a value in the right range.
    return int(math.floor(rightMin + (valueScaled * rightSpan))) def get_file_type(file_path):
    mime = magic.open(magic.MAGIC_MIME)
    mime.load()
    return mime.file(file_path) def get_file_size(file_path):
    stat = os.stat(file_path)
    return int(stat.st_size) class Logger:

    def __init__(self, log_name, log_mode):
        logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s')
        self.mode = log_mode
        self.log = logging.getLogger(log_name)
        self.log.setLevel(log_mode)

    def set_log_level(self, level="DEBUG"):
        self.log.setLevel(level)

    def message(self, message):
        if self.mode == "DEBUG":
            self.log.debug(message)
        if self.mode == "ERROR":
            self.log.error(message)
        if self.mode == "INFO":
            self.log.info(message) ICONS = {'smtp': "circle_blue.png", 'http':"circle_orange.png", 'dns': "circle_red.png", 'unknown': 'circle_gray.png'}
import urllib2
import tempfile
import os
import datetime
import cgi
import random
from django.http import Http404, HttpResponse
from django.utils import simplejson as json
from django.shortcuts import render_to_response
from django.shortcuts import redirect
from django.core.urlresolvers import reverse
from django.template.context import RequestContext
from django.conf import settings
from ovizart.pcap.forms import UploadPcapForm
from ovizart.modules.file.handler import Handler as FileHandler
from ovizart.modules.traffic.pcap.handler import Handler as PcapHandler
from ovizart.modules.traffic.flow.handler import Handler as FlowHandler
from ovizart.modules.traffic.parser.tcp.handler import Handler as TcpHandler
from ovizart.modules.traffic.parser.udp.handler import Handler as UDPHandler
from ovizart.pcap.models import UserJSonFile
from ovizart.modules.md5.handler import Handler as HashHandler
from ovizart.pcap.models import Flow, Pcap, PacketDetails, FlowDetails, HTTPDetails, DNSRequest, DNSResponse, SMTPDetails
from ovizart.modules.utils.handler import translate_time, get_file_type, get_file_size
from ovizart.modules.traffic.log.logger import Logger
from django.contrib.auth.decorators import login_required
from ovizart.api.constants import  ICONS
from django.core.paginator import Paginator, InvalidPage, EmptyPage

@login_required()
def upload(request):
    log = Logger("Upload form", "DEBUG")
    context = {
        'page_title': 'Upload your pcap file here',
        'upload_status': False,
        'message': request.session.get('message', False)
    }
    if request.method == "POST":
        form = UploadPcapForm(request.POST, request.FILES)
        if form.is_valid():
            user_id = request.user.id
            context['form'] = form
            file_handler = FileHandler()
            file_handler.create_dir()
            mem_file = request.FILES['pcap_file']
            log.message("file: %s" % mem_file.name)
            file_handler.save_file(mem_file)
            context['upload_status'] = True

            #save the file name to the db
            pcap_name = mem_file.name
            upload_path = file_handler.upload_dir
            # evey pcap file is saved as a flow container, there may or may not be flows, the pcaps colon will give the flow pcaps
            hash_handler = HashHandler()
            hash_value = hash_handler.get_hash(os.path.join(upload_path, pcap_name))
            request.session['uploaded_hash'] = hash_value
            request.session['uploaded_file_name'] = pcap_name
            # send the file to the defined protocol handler so that it can detect
            protocol_handler = settings.PROTOCOL_HANDLER
            package = "ovizart.modules.traffic.detector"
            module_name = ".".join([package, protocol_handler])
            # from ovizart.modules.traffic.detector.x import handler as imported_module
            traffic_detector_module = getattr(__import__(module_name, fromlist=["handler"]), "handler")
            traffic_detector_handler = traffic_detector_module.Handler()
            traffic_detector_handler.create_reassemble_information(file_handler.file_path, upload_path)
            output = traffic_detector_handler.detect_proto(file_handler.file_path, upload_path)

            if output == False:
                request.session['message'] = "Error occured. Please try again."
                return redirect('/pcap/upload')


            file_type = get_file_type(file_handler.file_path)
            file_size = get_file_size(file_handler.file_path)
            flow_file, created = Flow.objects.get_or_create(user_id=user_id, hash_value=hash_value,file_name=pcap_name,
                                                            path=upload_path, upload_time=datetime.datetime.now(),
                                                            file_type=file_type, file_size=file_size)

            if "tcp" in output:
                log.message("protocol detected: %s" % "TCP")
                # run tcp flow extractor
                p_read_handler = PcapHandler()
                p_read_handler.open_file(file_handler.file_path)
                p_read_handler.open_pcap()

                f_handler = FlowHandler(p_read_handler)
                flow, direction = f_handler.get_tcp_flows()

                p_write_handler = PcapHandler()
                files = f_handler.save_flow(flow, p_write_handler, save_path=upload_path)

                # save the flow pcap names to the mongo db
                pcap_list = map(lambda x: Pcap.objects.create(hash_value=hash_handler.get_hash(os.path.join(upload_path, x)), file_name=x, path=upload_path), files.values()[0])
                if flow_file.pcaps:
                    pre_li = flow_file.pcaps
                    pre_li.extend(pcap_list)
                    flow_file.pcaps = pre_li
                else:
                    flow_file.pcaps = pcap_list
                flow_file.save()

                p_read_handler.close_file()
                p_write_handler.close_file()
                # now i should hook a protocol detector
                # before that i should detect the application level protocol
                for f in files.values()[0]:
                    packets  = []
                    # better to save tcp level information to db here
                    full_path = os.path.join(upload_path, f)
                    p_read_handler.open_file(full_path)
                    p_read_handler.open_pcap()
                    pcap = p_read_handler.get_pcap()
                    # the list that will keep the tcp part of the packet
                    tcp_list = []
                    for ts, buf in pcap:
                        tcp_handler = TcpHandler()
                        tcp = tcp_handler.read_tcp(ts, buf)
                        # this list will be used at the layers above tcp
                        if tcp:
                            tcp_list.append((tcp, tcp_handler.ident))
                        else: continue
                        tcp_data = u"."
                        if tcp_handler.data:
                            tcp_data = tcp_handler.data
                            # some requests include hexadecimal info, most probably some binary info that can not be
                            # converted to the utf8, for now i better remove them, #TODO should handle them, though
                            # try with 4, tcp.data has binary request
                            # def get_tcp(n):
                            #    count = 1
                            #    f = file("milliyet.pcap", "rb")
                            #    reader = dpkt.pcap.Reader(f)
                            #    for ts, buf in reader:
                            #        if count == n:
                            #            f.close()
                            #            return buf
                            #        count += 1

                            data_li = tcp_data.split("\r\n")
                            tmp = []
                            for data in data_li:
                                try:
                                    data.encode("utf-8")
                                    tmp.append(data)
                                except:
                                    tmp.append("data that can not be encoded to utf-8")

                            tcp_data = " \n".join(tmp)


                        packet = PacketDetails.objects.create(ident=tcp_handler.ident, flow_hash=hash_value, timestamp=tcp_handler.timestamp,