{"action":"created","comment":{"url":"https://api.github.com/repos/inliniac/suricata/pulls/comments/51350988","id":51350988,"diff_hunk":"@@ -0,0 +1,150 @@\n+/* Copyright (C) 2007-2014 Open Information Security Foundation\n+ *\n+ * You can copy, redistribute or modify this Program under the terms of\n+ * the GNU General Public License version 2 as published by the Free\n+ * Software Foundation.\n+ *\n+ * This program is distributed in the hope that it will be useful,\n+ * but WITHOUT ANY WARRANTY; without even the implied warranty of\n+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+ * GNU General Public License for more details.\n+ *\n+ * You should have received a copy of the GNU General Public License\n+ * version 2 along with this program; if not, write to the Free Software\n+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA\n+ * 02110-1301, USA.\n+ */\n+\n+\n+/**\n+ * \\file\n+ *\n+ * \\author Mat Oldham <mat.oldham@gmail.com>\n+ *\n+ * Heap related functions for usage within TimeMachine\n+ */\n+\n+#include \"suricata-common.h\"\n+#include \"timemachine.h\"\n+#include \"timemachine-heap.h\"\n+#include \"timemachine-packet.h\"\n+\n+TimeMachineHeap* TimeMachineHeapNew(TimeMachineThreadVars* tmtv, TimeMachineHeapConf* conf) {\n+\n+    TimeMachineHeap *heap = SCMalloc(sizeof(TimeMachineHeap));  \n+    if (unlikely(heap == NULL)) {\n+        return NULL;     \n+    }\n+\n+    heap->conf = conf;\n+    heap->used_pool_count = 0;\n+    heap->unused_pool_count = 0;\n+    \n+    TAILQ_INIT(&heap->used_mem_pools);\n+    TAILQ_INIT(&heap->unused_mem_pools);\n+    TAILQ_INIT(&heap->unused_packets);    \n+    \n+    uint32_t expanded = TimeMachineHeapExpand(tmtv, heap, conf->prealloc_count);\n+    if (expanded != conf->prealloc_count) {\n+        SCLogError(SC_ERR_MEM_ALLOC, \"Error: Could not allocate initial memory for \"\n+                   \"time machine, expanded heap by: %\" PRIu32 \", requested: %\" PRIu32 \n+                   \" current Memory: %\" PRIu64 \" max Memory: %\" PRIu64, expanded, \n+                   conf->prealloc_count, tmtv->current_memory, timemachine_config.max_memory);\n+        exit(EXIT_FAILURE);                         \n+    }\n+    \n+    return heap;\n+}\n+\n+void TimeMachineHeapDestroy(TimeMachineHeap* heap) {\n+    \n+    if (heap == NULL) {\n+        return;\n+    }\n+    \n+    /* delete all the used pool data */\n+    while (heap->used_pool_count > 0) {\n+        TimeMachineMemPool* mem_pool;\n+            \n+        mem_pool = TAILQ_FIRST(&heap->used_mem_pools);\n+        TAILQ_REMOVE(&heap->used_mem_pools, mem_pool, next);\n+        SCFree(mem_pool);\n+\n+        heap->used_pool_count--;    \n+    }\n+    \n+    /* delete all the unused pool data */\n+    while (heap->unused_pool_count > 0) {\n+        TimeMachinePacket* packet;\n+        TimeMachineMemPool* mem_pool;\n+\n+        packet = TAILQ_FIRST(&heap->unused_packets);\n+        TAILQ_REMOVE(&heap->unused_packets, packet, next);\n+        TimeMachinePacketDestroy(packet);\n+            \n+        mem_pool = TAILQ_FIRST(&heap->unused_mem_pools);\n+        TAILQ_REMOVE(&heap->unused_mem_pools, mem_pool, next);\n+        SCFree(mem_pool);\n+                \n+        heap->unused_pool_count--;\n+    }    \n+}\n+\n+uint32_t TimeMachineHeapExpand(TimeMachineThreadVars* tmtv, TimeMachineHeap* heap, uint32_t expand_by) {\n+    uint32_t i;\n+           \n+    if (heap == NULL) {\n+        return 0;\n+    }\n+\n+    uint32_t expand_count = expand_by;\n+    uint32_t expand_mem_size = sizeof(TimeMachineMemPool) + heap->conf->max_packet_size + \n+                      sizeof(TimeMachinePacket);\n+                   \n+    if ((expand_mem_size * expand_by) + tmtv->current_memory > timemachine_config.max_memory) {\n+        expand_count = (timemachine_config.max_memory - tmtv->current_memory) / expand_mem_size + 1;\n+    }    \n+    \n+    for (i = 0; i < expand_count; i++) {\n+        TimeMachineMemPool* mem_pool;\n+        TimeMachinePacket* packet;\n+        \n+        mem_pool = SCCalloc(sizeof(TimeMachineMemPool), 1);\n+        if (mem_pool == NULL) {\n+            return i;\n+        }    \n+        \n+        mem_pool->mem = SCMalloc(heap->conf->max_packet_size);\n+        if (mem_pool->mem == NULL) {\n+            SCFree(mem_pool);\n+            return i;\n+        }\n+        \n+        packet = TimeMachinePacketNew();","path":"src/timemachine-heap.c","position":123,"original_position":123,"commit_id":"832bee97bd7faef03e67418b39b1b171079aa7f5","original_commit_id":"832bee97bd7faef03e67418b39b1b171079aa7f5","user":{"login":"inliniac","id":719366,"avatar_url":"https://avatars.githubusercontent.com/u/719366?v=3","gravatar_id":"","url":"https://api.github.com/users/inliniac","html_url":"https://github.com/inliniac","followers_url":"https://api.github.com/users/inliniac/followers","following_url":"https://api.github.com/users/inliniac/following{/other_user}","gists_url":"https://api.github.com/users/inliniac/gists{/gist_id}","starred_url":"https://api.github.com/users/inliniac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inliniac/subscriptions","organizations_url":"https://api.github.com/users/inliniac/orgs","repos_url":"https://api.github.com/users/inliniac/repos","events_url":"https://api.github.com/users/inliniac/events{/privacy}","received_events_url":"https://api.github.com/users/inliniac/received_events","type":"User","site_admin":false},"body":"what is packet used for?","created_at":"2016-01-30T21:11:09Z","updated_at":"2016-01-30T21:11:09Z","html_url":"https://github.com/inliniac/suricata/pull/1817#discussion_r51350988","pull_request_url":"https://api.github.com/repos/inliniac/suricata/pulls/1817","_links":{"self":{"href":"https://api.github.com/repos/inliniac/suricata/pulls/comments/51350988"},"html":{"href":"https://github.com/inliniac/suricata/pull/1817#discussion_r51350988"},"pull_request":{"href":"https://api.github.com/repos/inliniac/suricata/pulls/1817"}}},"pull_request":{"url":"https://api.github.com/repos/inliniac/suricata/pulls/1817","id":57230873,"html_url":"https://github.com/inliniac/suricata/pull/1817","diff_url":"https://github.com/inliniac/suricata/pull/1817.diff","patch_url":"https://github.com/inliniac/suricata/pull/1817.patch","issue_url":"https://api.github.com/repos/inliniac/suricata/issues/1817","number":1817,"state":"open","locked":false,"title":"Capture full session on alert - Feature #120","user":{"login":"meatchicken","id":5770746,"avatar_url":"https://avatars.githubusercontent.com/u/5770746?v=3","gravatar_id":"","url":"https://api.github.com/users/meatchicken","html_url":"https://github.com/meatchicken","followers_url":"https://api.github.com/users/meatchicken/followers","following_url":"https://api.github.com/users/meatchicken/following{/other_user}","gists_url":"https://api.github.com/users/meatchicken/gists{/gist_id}","starred_url":"https://api.github.com/users/meatchicken/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meatchicken/subscriptions","organizations_url":"https://api.github.com/users/meatchicken/orgs","repos_url":"https://api.github.com/users/meatchicken/repos","events_url":"https://api.github.com/users/meatchicken/events{/privacy}","received_events_url":"https://api.github.com/users/meatchicken/received_events","type":"User","site_admin":false},"body":"In most signature based IDS's, when a signature fires, they have the\r\nability to dump the packet data for which the signature fired upon\r\nand subsequent packet data, usually within the same network stream.\r\nUnfortunately, this can mean important information may not be\r\navailable to a user if the information occured prior to the packet\r\ncontaining the signature.  For example, if a signature fires on an\r\nHTTP response, it is possible that important information in an\r\nHTTP request may not be available but could be useful in\r\ninvestigations.\r\n\r\nTimeMachine is a fast, memory-efficient solution for storing network\r\ntraffic for retrieval after an alarm fires and dumping all stored\r\ncaptured traffic to disk.\r\n\r\nWhen TimeMachine is enabled, the flow manager stores all packets\r\ncontaining data in a set of pre-allocated in-memory heaps based\r\non a configurable packet size and configurable amount of memory\r\nto use.\r\n\r\nWhen a new packet is processed by a given thread, several actions may\r\ntake place:\r\n\r\n  -  In cases where TimeMachine does not know of an existing flow,\r\n     the flow manager will create a new flow and associate the given\r\n     packet to the flow in one of the pre-allocated in-memory heaps.\r\n\r\n  -  In cases where TimeMachine already knows of an existing flow,\r\n     it once again associates the packet to one of the pre-allocated\r\n     in-memory heaps and associates the packet to the given flow by\r\n     pushing the packet to the end of the queue of stored packets.\r\n\r\n  -  In cases where the packet is associated with a given alert\r\n     generated by the detection engine, a new output file will be\r\n     opened and all data associated with the packet's flow will be\r\n     dumped to disk in chronological order.\r\n\r\n  -  In cases where an output file already exists indicating the\r\n     flow was previously associated with an alert, packets are not\r\n     stored within the in-memory heap and instead are written\r\n     directly to disk.\r\n\r\nAs the heaps are a fixed size, when a given heap is full, TimeMachine\r\nwill purge the oldest packet first.  While the heaps are\r\npre-allocated, configuration parameters allow for the heaps to grow\r\nas a one-size-fits-all solution doesn't work for packet sizes.\r\n\r\nTimeMachine is an effective alternative in cases where full packet\r\ncapture is not available due to disk or IOP's constraints.  The\r\noverall effectiveness of TimeMachine is dependent upon the overall\r\nnetwork speed, size of heaps configured for TimeMachine and total\r\nmemory available.\r\n\r\nTo enable timemachine, set \"enabled: yes\" under timemachine in\r\nsuricata.yaml.\r\n\r\nThis is an implementation to address feature request #120, capture\r\nfull session on alert:\r\n\r\nhttps://redmine.openinfosecfoundation.org/issues/120","created_at":"2016-01-26T16:04:13Z","updated_at":"2016-01-30T21:11:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"af1bbbb4aaaf765a285fd824dc5f777ce1ae57be","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/inliniac/suricata/pulls/1817/commits","review_comments_url":"https://api.github.com/repos/inliniac/suricata/pulls/1817/comments","review_comment_url":"https://api.github.com/repos/inliniac/suricata/pulls/comments{/number}","comments_url":"https://api.github.com/repos/inliniac/suricata/issues/1817/comments","statuses_url":"https://api.github.com/repos/inliniac/suricata/statuses/832bee97bd7faef03e67418b39b1b171079aa7f5","head":{"label":"meatchicken:timemachine","ref":"timemachine","sha":"832bee97bd7faef03e67418b39b1b171079aa7f5","user":{"login":"meatchicken","id":5770746,"avatar_url":"https://avatars.githubusercontent.com/u/5770746?v=3","gravatar_id":"","url":"https://api.github.com/users/meatchicken","html_url":"https://github.com/meatchicken","followers_url":"https://api.github.com/users/meatchicken/followers","following_url":"https://api.github.com/users/meatchicken/following{/other_user}","gists_url":"https://api.github.com/users/meatchicken/gists{/gist_id}","starred_url":"https://api.github.com/users/meatchicken/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meatchicken/subscriptions","organizations_url":"https://api.github.com/users/meatchicken/orgs","repos_url":"https://api.github.com/users/meatchicken/repos","events_url":"https://api.github.com/users/meatchicken/events{/privacy}","received_events_url":"https://api.github.com/users/meatchicken/received_events","type":"User","site_admin":false},"repo":{"id":43271711,"name":"suricata","full_name":"meatchicken/suricata","owner":{"login":"meatchicken","id":5770746,"avatar_url":"https://avatars.githubusercontent.com/u/5770746?v=3","gravatar_id":"","url":"https://api.github.com/users/meatchicken","html_url":"https://github.com/meatchicken","followers_url":"https://api.github.com/users/meatchicken/followers","following_url":"https://api.github.com/users/meatchicken/following{/other_user}","gists_url":"https://api.github.com/users/meatchicken/gists{/gist_id}","starred_url":"https://api.github.com/users/meatchicken/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meatchicken/subscriptions","organizations_url":"https://api.github.com/users/meatchicken/orgs","repos_url":"https://api.github.com/users/meatchicken/repos","events_url":"https://api.github.com/users/meatchicken/events{/privacy}","received_events_url":"https://api.github.com/users/meatchicken/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/meatchicken/suricata","description":"Mirror of the official OISF Suricata git repository","fork":true,"url":"https://api.github.com/repos/meatchicken/suricata","forks_url":"https://api.github.com/repos/meatchicken/suricata/forks","keys_url":"https://api.github.com/repos/meatchicken/suricata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/meatchicken/suricata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/meatchicken/suricata/teams","hooks_url":"https://api.github.com/repos/meatchicken/suricata/hooks","issue_events_url":"https://api.github.com/repos/meatchicken/suricata/issues/events{/number}","events_url":"https://api.github.com/repos/meatchicken/suricata/events","assignees_url":"https://api.github.com/repos/meatchicken/suricata/assignees{/user}","branches_url":"https://api.github.com/repos/meatchicken/suricata/branches{/branch}","tags_url":"https://api.github.com/repos/meatchicken/suricata/tags","blobs_url":"https://api.github.com/repos/meatchicken/suricata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/meatchicken/suricata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/meatchicken/suricata/git/refs{/sha}","trees_url":"https://api.github.com/repos/meatchicken/suricata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/meatchicken/suricata/statuses/{sha}","languages_url":"https://api.github.com/repos/meatchicken/suricata/languages","stargazers_url":"https://api.github.com/repos/meatchicken/suricata/stargazers","contributors_url":"https://api.github.com/repos/meatchicken/suricata/contributors","subscribers_url":"https://api.github.com/repos/meatchicken/suricata/subscribers","subscription_url":"https://api.github.com/repos/meatchicken/suricata/subscription","commits_url":"https://api.github.com/repos/meatchicken/suricata/commits{/sha}","git_commits_url":"https://api.github.com/repos/meatchicken/suricata/git/commits{/sha}","comments_url":"https://api.github.com/repos/meatchicken/suricata/comments{/number}","issue_comment_url":"https://api.github.com/repos/meatchicken/suricata/issues/comments{/number}","contents_url":"https://api.github.com/repos/meatchicken/suricata/contents/{+path}","compare_url":"https://api.github.com/repos/meatchicken/suricata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/meatchicken/suricata/merges","archive_url":"https://api.github.com/repos/meatchicken/suricata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/meatchicken/suricata/downloads","issues_url":"https://api.github.com/repos/meatchicken/suricata/issues{/number}","pulls_url":"https://api.github.com/repos/meatchicken/suricata/pulls{/number}","milestones_url":"https://api.github.com/repos/meatchicken/suricata/milestones{/number}","notifications_url":"https://api.github.com/repos/meatchicken/suricata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/meatchicken/suricata/labels{/name}","releases_url":"https://api.github.com/repos/meatchicken/suricata/releases{/id}","deployments_url":"https://api.github.com/repos/meatchicken/suricata/deployments","created_at":"2015-09-28T00:33:26Z","updated_at":"2015-09-28T00:33:35Z","pushed_at":"2016-01-27T06:18:56Z","git_url":"git://github.com/meatchicken/suricata.git","ssh_url":"git@github.com:meatchicken/suricata.git","clone_url":"https://github.com/meatchicken/suricata.git","svn_url":"https://github.com/meatchicken/suricata","homepage":"http://suricata-ids.org/","size":36365,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"inliniac:master","ref":"master","sha":"202b11c0f6215bfee120addf651a7dc30a98f0e5","user":{"login":"inliniac","id":719366,"avatar_url":"https://avatars.githubusercontent.com/u/719366?v=3","gravatar_id":"","url":"https://api.github.com/users/inliniac","html_url":"https://github.com/inliniac","followers_url":"https://api.github.com/users/inliniac/followers","following_url":"https://api.github.com/users/inliniac/following{/other_user}","gists_url":"https://api.github.com/users/inliniac/gists{/gist_id}","starred_url":"https://api.github.com/users/inliniac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inliniac/subscriptions","organizations_url":"https://api.github.com/users/inliniac/orgs","repos_url":"https://api.github.com/users/inliniac/repos","events_url":"https://api.github.com/users/inliniac/events{/privacy}","received_events_url":"https://api.github.com/users/inliniac/received_events","type":"User","site_admin":false},"repo":{"id":5410234,"name":"suricata","full_name":"inliniac/suricata","owner":{"login":"inliniac","id":719366,"avatar_url":"https://avatars.githubusercontent.com/u/719366?v=3","gravatar_id":"","url":"https://api.github.com/users/inliniac","html_url":"https://github.com/inliniac","followers_url":"https://api.github.com/users/inliniac/followers","following_url":"https://api.github.com/users/inliniac/following{/other_user}","gists_url":"https://api.github.com/users/inliniac/gists{/gist_id}","starred_url":"https://api.github.com/users/inliniac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inliniac/subscriptions","organizations_url":"https://api.github.com/users/inliniac/orgs","repos_url":"https://api.github.com/users/inliniac/repos","events_url":"https://api.github.com/users/inliniac/events{/privacy}","received_events_url":"https://api.github.com/users/inliniac/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/inliniac/suricata","description":"Mirror of the official OISF Suricata git repository","fork":false,"url":"https://api.github.com/repos/inliniac/suricata","forks_url":"https://api.github.com/repos/inliniac/suricata/forks","keys_url":"https://api.github.com/repos/inliniac/suricata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/inliniac/suricata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/inliniac/suricata/teams","hooks_url":"https://api.github.com/repos/inliniac/suricata/hooks","issue_events_url":"https://api.github.com/repos/inliniac/suricata/issues/events{/number}","events_url":"https://api.github.com/repos/inliniac/suricata/events","assignees_url":"https://api.github.com/repos/inliniac/suricata/assignees{/user}","branches_url":"https://api.github.com/repos/inliniac/suricata/branches{/branch}","tags_url":"https://api.github.com/repos/inliniac/suricata/tags","blobs_url":"https://api.github.com/repos/inliniac/suricata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/inliniac/suricata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/inliniac/suricata/git/refs{/sha}","trees_url":"https://api.github.com/repos/inliniac/suricata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/inliniac/suricata/statuses/{sha}","languages_url":"https://api.github.com/repos/inliniac/suricata/languages","stargazers_url":"https://api.github.com/repos/inliniac/suricata/stargazers","contributors_url":"https://api.github.com/repos/inliniac/suricata/contributors","subscribers_url":"https://api.github.com/repos/inliniac/suricata/subscribers","subscription_url":"https://api.github.com/repos/inliniac/suricata/subscription","commits_url":"https://api.github.com/repos/inliniac/suricata/commits{/sha}","git_commits_url":"https://api.github.com/repos/inliniac/suricata/git/commits{/sha}","comments_url":"https://api.github.com/repos/inliniac/suricata/comments{/number}","issue_comment_url":"https://api.github.com/repos/inliniac/suricata/issues/comments{/number}","contents_url":"https://api.github.com/repos/inliniac/suricata/contents/{+path}","compare_url":"https://api.github.com/repos/inliniac/suricata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/inliniac/suricata/merges","archive_url":"https://api.github.com/repos/inliniac/suricata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/inliniac/suricata/downloads","issues_url":"https://api.github.com/repos/inliniac/suricata/issues{/number}","pulls_url":"https://api.github.com/repos/inliniac/suricata/pulls{/number}","milestones_url":"https://api.github.com/repos/inliniac/suricata/milestones{/number}","notifications_url":"https://api.github.com/repos/inliniac/suricata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/inliniac/suricata/labels{/name}","releases_url":"https://api.github.com/repos/inliniac/suricata/releases{/id}","deployments_url":"https://api.github.com/repos/inliniac/suricata/deployments","created_at":"2012-08-14T08:30:06Z","updated_at":"2016-01-29T09:59:34Z","pushed_at":"2016-01-29T21:03:21Z","git_url":"git://github.com/inliniac/suricata.git","ssh_url":"git@github.com:inliniac/suricata.git","clone_url":"https://github.com/inliniac/suricata.git","svn_url":"https://github.com/inliniac/suricata","homepage":"http://suricata-ids.org/","size":52467,"stargazers_count":231,"watchers_count":231,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":216,"mirror_url":null,"open_issues_count":43,"forks":216,"open_issues":43,"watchers":231,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/inliniac/suricata/pulls/1817"},"html":{"href":"https://github.com/inliniac/suricata/pull/1817"},"issue":{"href":"https://api.github.com/repos/inliniac/suricata/issues/1817"},"comments":{"href":"https://api.github.com/repos/inliniac/suricata/issues/1817/comments"},"review_comments":{"href":"https://api.github.com/repos/inliniac/suricata/pulls/1817/comments"},"review_comment":{"href":"https://api.github.com/repos/inliniac/suricata/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/inliniac/suricata/pulls/1817/commits"},"statuses":{"href":"https://api.github.com/repos/inliniac/suricata/statuses/832bee97bd7faef03e67418b39b1b171079aa7f5"}}}}