init logdecoder app files

app.py

@@ -0,0 +1,314 @@
+# To run streamlit, go to terminal and type: 'streamlit run app.py'
+# Core Packages ###########################
+import streamlit as st
+import os
+
+import time
+import numpy as np
+import pandas as pd
+
+from loglizer.models import PCA
+from loglizer import dataloader, preprocessing
+import openai
+
+openai.api_key = os.environ["FREEEDU_OPENAI_API_KEY"]
+
+project_title = "ChatGPT Log Decoder"
+project_desc = """
+Intrusion Detection Systems (IDS) are powerful security tools that monitor network traffic for suspicious activity and issues alerts when such activities are discovered. 
+While these systems are commonly used by cybersecurity professionals and IT experts, 
+the technical jargon and analysis that an IDS provides can be incomprehensible to the average user. 
+This project aims to bridge this gap by utilizing ChatGPT's advanced natural language processing to 
+articulate the technical information received from an IDS into human-readable form, allowing common everyday users 
+to grasp the extent of their network's security status without the need for a technical background.
+"""
+
+project_link = "https://github.com/logpai/loglizer"
+project_icon = "46_Heatmap.png"
+st.set_page_config(page_title=project_title, initial_sidebar_state='collapsed', page_icon=project_icon)
+
+# additional info from the readme
+add_info_md = """
+
+# loglizer
+
+
+**Loglizer is a machine learning-based log analysis toolkit for automated anomaly detection**. 
+  
+
+Logs are imperative in the development and maintenance process of many software systems. They record detailed
+runtime information during system operation that allows developers and support engineers to monitor their systems and track abnormal behaviors and errors. Loglizer provides a toolkit that implements a number of machine-learning based log analysis techniques for automated anomaly detection. 
+
+:telescope: If you use loglizer in your research for publication, please kindly cite the following paper.
++ Shilin He, Jieming Zhu, Pinjia He, Michael R. Lyu. [Experience Report: System Log Analysis for Anomaly Detection](https://jiemingzhu.github.io/pub/slhe_issre2016.pdf), *IEEE International Symposium on Software Reliability Engineering (ISSRE)*, 2016. [[Bibtex](https://dblp.org/rec/bibtex/conf/issre/HeZHL16)][[中文版本](https://github.com/AmateurEvents/article/issues/2)]
+**(ISSRE Most Influential Paper)**
+
+## Framework
+
+![Framework of Anomaly Detection](/docs/img/framework.png)
+
+The log analysis framework for anomaly detection usually comprises the following components:
+
+1. **Log collection:** Logs are generated at runtime and aggregated into a centralized place with a data streaming pipeline, such as Flume and Kafka. 
+2. **Log parsing:** The goal of log parsing is to convert unstructured log messages into a map of structured events, based on which sophisticated machine learning models can be applied. The details of log parsing can be found at [our logparser project](https://github.com/logpai/logparser).
+3. **Feature extraction:** Structured logs can be sliced into short log sequences through interval window, sliding window, or session window. Then, feature extraction is performed to vectorize each log sequence, for example, using an event counting vector. 
+4. **Anomaly detection:** Anomaly detection models are trained to check whether a given feature vector is an anomaly or not.
+
+
+## Models
+
+Anomaly detection models currently available:
+
+| Model | Paper reference |
+| :--- | :--- |
+| **Supervised models** |
+| LR | [**EuroSys'10**] [Fingerprinting the Datacenter: Automated Classification of Performance Crises](https://www.microsoft.com/en-us/research/wp-content/uploads/2009/07/hiLighter.pdf), by Peter Bodík, Moises Goldszmidt, Armando Fox, Hans Andersen. [**Microsoft**] |
+| Decision Tree | [**ICAC'04**] [Failure Diagnosis Using Decision Trees](http://www.cs.berkeley.edu/~brewer/papers/icac2004_chen_diagnosis.pdf), by Mike Chen, Alice X. Zheng, Jim Lloyd, Michael I. Jordan, Eric Brewer. [**eBay**] |
+| SVM | [**ICDM'07**] [Failure Prediction in IBM BlueGene/L Event Logs](https://www.researchgate.net/publication/4324148_Failure_Prediction_in_IBM_BlueGeneL_Event_Logs), by Yinglung Liang, Yanyong Zhang, Hui Xiong, Ramendra Sahoo. [**IBM**]|
+| **Unsupervised models** |
+| LOF | [**SIGMOD'00**] [LOF: Identifying Density-Based Local Outliers](), by Markus M. Breunig, Hans-Peter Kriegel, Raymond T. Ng, Jörg Sander. |
+| One-Class SVM | [**Neural Computation'01**] [Estimating the Support of a High-Dimensional Distribution](), by John Platt, Bernhard Schölkopf, John Shawe-Taylor, Alex J. Smola, Robert C. Williamson. |
+| Isolation Forest | [**ICDM'08**] [Isolation Forest](https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf), by Fei Tony Liu, Kai Ming Ting, Zhi-Hua Zhou. |
+| PCA | [**SOSP'09**] [Large-Scale System Problems Detection by Mining Console Logs](http://iiis.tsinghua.edu.cn/~weixu/files/sosp09.pdf), by Wei Xu, Ling Huang, Armando Fox, David Patterson, Michael I. Jordan. [**Intel**] |
+| Invariants Mining | [**ATC'10**] [Mining Invariants from Console Logs for System Problem Detection](https://www.usenix.org/legacy/event/atc10/tech/full_papers/Lou.pdf), by Jian-Guang Lou, Qiang Fu, Shengqi Yang, Ye Xu, Jiang Li. [**Microsoft**]|
+| Clustering | [**ICSE'16**] [Log Clustering based Problem Identification for Online Service Systems](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/07/ICSE-2016-2-Log-Clustering-based-Problem-Identification-for-Online-Service-Systems.pdf), by Qingwei Lin, Hongyu Zhang, Jian-Guang Lou, Yu Zhang, Xuewei Chen. [**Microsoft**]|
+| DeepLog (coming)| [**CCS'17**] [DeepLog: Anomaly Detection and Diagnosis from System Logs through Deep Learning](https://www.cs.utah.edu/~lifeifei/papers/deeplog.pdf), by Min Du, Feifei Li, Guineng Zheng, Vivek Srikumar. |
+| AutoEncoder (coming)| [**Arxiv'18**] [Anomaly Detection using Autoencoders in High Performance Computing Systems](https://arxiv.org/abs/1811.05269), by Andrea Borghesi, Andrea Bartolini, Michele Lombardi, Michela Milano, Luca Benini. |
+
+
+## Log data
+We have collected a set of labeled log datasets in [loghub](https://github.com/logpai/loghub) for research purposes. If you are interested in the datasets, please follow the link to submit your access request.
+
+## Install
+```bash
+git clone https://github.com/logpai/loglizer.git
+cd loglizer
+pip install -r requirements.txt
+```
+
+## API usage
+
+```python
+# Load HDFS dataset. If you would like to try your own log, you need to rewrite the load function.
+(x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(...)
+
+# Feature extraction and transformation
+feature_extractor = preprocessing.FeatureExtractor()
+feature_extractor.fit_transform(...) 
+
+# Model training
+model = PCA()
+model.fit(...)
+
+# Feature transform after fitting
+x_test = feature_extractor.transform(...)
+# Model evaluation with labeled data
+model.evaluate(...)
+
+# Anomaly prediction
+x_test = feature_extractor.transform(...)
+model.predict(...) # predict anomalies on given data
+```
+
+For more details, please follow [the demo](./docs/demo.md) in the docs to get started. Please note that all ML models are not magic, you need to figure out how to tune the parameters in order to make them work on your own data. 
+
+## Benchmarking results 
+
+If you would like to reproduce the following results, please run [benchmarks/HDFS_bechmark.py](./benchmarks/HDFS_bechmark.py) on the full HDFS dataset (HDFS100k is for demo only).
+
+|       |            | HDFS |     |
+| :----:|:----:|:----:|:----:|
+| **Model** | **Precision** | **Recall** | **F1** |
+| LR| 0.955 |	0.911 |	0.933 |
+| Decision Tree | 0.998 |	0.998 |	0.998 |
+| SVM| 0.959 |	0.970 |	0.965 |
+| LOF | 0.967 | 0.561 | 0.710 |
+| One-Class SVM | 0.995 | 0.222| 0.363 |
+| Isolation Forest |  0.830 | 0.776 | 0.802 |
+| PCA | 0.975 | 0.635 | 0.769|
+| Invariants Mining | 0.888 | 0.945 | 0.915|
+| Clustering | 1.000 | 0.720 | 0.837 |
+
+## Contributors
++ [Shilin He](https://shilinhe.github.io), The Chinese University of Hong Kong
++ [Jieming Zhu](https://jiemingzhu.github.io), The Chinese University of Hong Kong, currently at Huawei Noah's Ark Lab
++ [Pinjia He](https://pinjiahe.github.io/), The Chinese University of Hong Kong, currently at ETH Zurich
+
+
+## Feedback
+For any questions or feedback, please post to [the issue page](https://github.com/logpai/loglizer/issues/new). 
+
+"""
+#######################################################################################################################
+default_log_path = os.path.join("data","HDFS","HDFS_100k.log_structured.csv")
+def load_loglizer(train_log_path=default_log_path):
+    print("Loading Loglizer PCA model:")
+    start = time.time()
+    (x_train, _), (_, _), _ = dataloader.load_HDFS(train_log_path, window='session',
+                                                split_type='sequential', save_csv=True)
+    feature_extractor = preprocessing.FeatureExtractor()
+    x_train = feature_extractor.fit_transform(x_train, term_weighting='tf-idf',
+                                              normalization='zero-mean')
+    model = PCA()
+    model.fit(x_train)
+
+    stop = time.time()
+    print("Loading complete. Time elapsed: " + str(stop - start))
+
+    # Extract the event labels for each unique eventid in the source structured log file
+    # Load the CSV file
+    df = pd.read_csv(train_log_path)
+
+    # Extract unique 'EventId' and 'EventTemplate' pairs
+    event_names = df[['EventId', 'EventTemplate']].drop_duplicates()
+    event_names_dict = dict(zip(event_names['EventId'], event_names['EventTemplate']))
+
+    return model, feature_extractor, event_names_dict
+
+def analyze_with_chatgpt(anomalous_packets):
+    context = {"role":"system",
+               "content":"""
+               You will receive 5 rows of transformed data logs, where the middlemost log is flagged as anomalous.
+               Your job is to analyze and compare this log with its surrounding context logs, and find out why it was flagged as anomalous. 
+               Your response should be 50 to 150 words only; be as concise as possible and address the user directly. 
+               Only mention the key issues and cause of the anomaly. Never mention the Column Names, the middlemost log, surrounding context log.
+               You can mention what the irregular values in a column represent, and base your report on that.
+               Assume that the user has no knowledge of networks or cybersecurity. Your response should start with
+                "Based on the logs,".
+               Try to avoid telling the user 'if the issue persists'.
+               Give simple step-by-step advice on what the user can do on their own.
+               If the advice is beyond the scope of the everyday user, notify them that the assistance of a professional 
+               is necessary for the level of intrusion you found.
+               """}
+    message_prompt = []
+    message_prompt.append(context)
+    anomalies = f"*Column Names*: {anomalous_packets.columns} --- "
+    # Build a sequence of messages, with each message being a single packet up to the last packet flagged as anomalous
+    for i,p in anomalous_packets.iterrows():
+        anomalies += f"*Row {i+1}*: "
+        anomalies += str(p.values)
+        anomalies += " --- "
+
+    message_format = {"role":"user",
+                      "content":anomalies}
+    message_prompt.append(message_format)
+
+    # Generate the response
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=message_prompt,
+        max_tokens=1008,
+        temperature=0.7,
+        n=1,
+        stop=None,
+    )
+
+    # Extract the response text from the API response
+    response_text = response['choices'][0]['message']['content']
+    # Return the response text and updated chat history
+    return response_text
+
+def main():
+    head_col = st.columns([1,8])
+    with head_col[0]:
+        st.image(project_icon)
+    with head_col[1]:
+        st.title(project_title)
+
+    st.write(project_desc)
+    st.write(f"Source Project: {project_link}")
+    expander = st.expander("Additional Information on the Source Project (Loglizer)")
+    expander.markdown(add_info_md)
+    st.markdown("***")
+    st.subheader("")
+#########################################
+
+    # instructions and file upload button
+    st.subheader("""
+    How to use: 
+     1. Upload your log file (must be .csv format)
+     2. Click the 'Run Loglizer' button
+                 """)
+    uploaded_file = st.file_uploader("Upload a log file in csv format:",
+                               type=["csv"],
+                               accept_multiple_files=False)
+
+#########################################
+
+    run_button = st.button("Run Loglizer")
+
+    if "anomalies" not in st.session_state:
+        st.session_state.anomalies = []
+    if "col_names" not in st.session_state:
+        st.session_state.col_names = []
+
+    # button is clicked
+    if run_button:
+        if uploaded_file is None: # if no files were uploaded:
+            st.error("Please upload a .csv log file")
+
+        else:
+            # Ensure temp directory exists
+            if not os.path.exists('temp'):
+                os.makedirs('temp')
+
+            filename = os.path.basename(uploaded_file.name)
+            file_path = os.path.join("temp", filename)
+            # Write out the uploaded file to temp directory
+            with open(file_path, 'wb') as f:
+                f.write(uploaded_file.getbuffer())
+
+            with st.spinner('Loading Loglizer...'):
+                model, feature_extractor, event_names_dict = load_loglizer()
+
+            # Simulate the 'live' log feed
+            (x_live, _), (_, _), _ = dataloader.load_HDFS(file_path, window='session', split_type='sequential')
+            x_live, st.session_state.col_names = feature_extractor.transform(x_live)
+            # st.write(f"x_live shape: {x_live.shape}")
+
+            # Map event IDs to event templates
+            st.session_state.col_names = [event_names_dict[id] if id in event_names_dict else id for id in
+                                          st.session_state.col_names]
+
+            for i in range(len(x_live)):
+                log = x_live[i]
+                log = np.expand_dims(log, axis=0)
+                prediction = model.predict(log)
+                print(f"prediction for x_live[{i}]: {prediction}")
+                if prediction == 1:
+                    # Anomaly detected
+                    context = x_live[max(0, i - 2):min(i + 3, len(x_live))]  # Get the two logs before and after
+                    st.session_state.anomalies.append(context)
+            # st.write("predict on x_live:")
+            # st.write(model.predict(x_live))
+
+    st.write("Anomalous events")
+    st.write(len(st.session_state.anomalies))
+    #
+    # st.write("Column names")
+    # st.write(st.session_state.col_names)
+
+    if len(st.session_state.anomalies) > 0:
+        # Displaying the anomalous packets and sending to ChatGPT
+        selected_index = st.selectbox('Select an anomaly', list(range(len(st.session_state.anomalies))), 0)
+        selected_anomaly = st.session_state.anomalies[selected_index]
+        st.write(f"Loglizer found and compiled {selected_anomaly.shape[1]} events from the logs.\n The middle entry has an anomaly:")
+        # st.write(f"Shape: {selected_anomaly.shape} Type: {type(selected_anomaly)}")
+
+        # Convert the numpy array to a pandas DataFrame
+        selected_anomaly_df = pd.DataFrame(selected_anomaly)
+        # Set the column names
+        selected_anomaly_df.columns = st.session_state.col_names
+
+        # st.write(selected_anomaly)
+        st.write(selected_anomaly_df)
+
+        if st.button('Send to ChatGPT'):
+            result = analyze_with_chatgpt(selected_anomaly_df)
+            st.write(result)
+
+
+if __name__ == '__main__':
+    main()
+
+# To run streamlit, go to terminal and type: 'streamlit run app-source.py'

data/HDFS/HDFS_100k.log_structured.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbb13cc937bb35ad683dfcd71929242d7b1e0c01e2a94f79c637847c30a42190
+size 20892087

data/HDFS/anomaly_label.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c711ed6c8848fc3243fb4d092f172f31d128c8a6ec7f26ebba72ab931885ed8
+size 18637554

data/app.py

@@ -0,0 +1,314 @@
+# To run streamlit, go to terminal and type: 'streamlit run app.py'
+# Core Packages ###########################
+import streamlit as st
+import os
+
+import time
+import numpy as np
+import pandas as pd
+
+from loglizer.models import PCA
+from loglizer import dataloader, preprocessing
+import openai
+
+openai.api_key = os.environ["FREEEDU_OPENAI_API_KEY"]
+
+project_title = "ChatGPT Log Decoder"
+project_desc = """
+Intrusion Detection Systems (IDS) are powerful security tools that monitor network traffic for suspicious activity and issues alerts when such activities are discovered. 
+While these systems are commonly used by cybersecurity professionals and IT experts, 
+the technical jargon and analysis that an IDS provides can be incomprehensible to the average user. 
+This project aims to bridge this gap by utilizing ChatGPT's advanced natural language processing to 
+articulate the technical information received from an IDS into human-readable form, allowing common everyday users 
+to grasp the extent of their network's security status without the need for a technical background.
+"""
+
+project_link = "https://github.com/logpai/loglizer"
+project_icon = "46_Heatmap.png"
+st.set_page_config(page_title=project_title, initial_sidebar_state='collapsed', page_icon=project_icon)
+
+# additional info from the readme
+add_info_md = """
+
+# loglizer
+
+
+**Loglizer is a machine learning-based log analysis toolkit for automated anomaly detection**. 
+  
+
+Logs are imperative in the development and maintenance process of many software systems. They record detailed
+runtime information during system operation that allows developers and support engineers to monitor their systems and track abnormal behaviors and errors. Loglizer provides a toolkit that implements a number of machine-learning based log analysis techniques for automated anomaly detection. 
+
+:telescope: If you use loglizer in your research for publication, please kindly cite the following paper.
++ Shilin He, Jieming Zhu, Pinjia He, Michael R. Lyu. [Experience Report: System Log Analysis for Anomaly Detection](https://jiemingzhu.github.io/pub/slhe_issre2016.pdf), *IEEE International Symposium on Software Reliability Engineering (ISSRE)*, 2016. [[Bibtex](https://dblp.org/rec/bibtex/conf/issre/HeZHL16)][[中文版本](https://github.com/AmateurEvents/article/issues/2)]
+**(ISSRE Most Influential Paper)**
+
+## Framework
+
+![Framework of Anomaly Detection](/docs/img/framework.png)
+
+The log analysis framework for anomaly detection usually comprises the following components:
+
+1. **Log collection:** Logs are generated at runtime and aggregated into a centralized place with a data streaming pipeline, such as Flume and Kafka. 
+2. **Log parsing:** The goal of log parsing is to convert unstructured log messages into a map of structured events, based on which sophisticated machine learning models can be applied. The details of log parsing can be found at [our logparser project](https://github.com/logpai/logparser).
+3. **Feature extraction:** Structured logs can be sliced into short log sequences through interval window, sliding window, or session window. Then, feature extraction is performed to vectorize each log sequence, for example, using an event counting vector. 
+4. **Anomaly detection:** Anomaly detection models are trained to check whether a given feature vector is an anomaly or not.
+
+
+## Models
+
+Anomaly detection models currently available:
+
+| Model | Paper reference |
+| :--- | :--- |
+| **Supervised models** |
+| LR | [**EuroSys'10**] [Fingerprinting the Datacenter: Automated Classification of Performance Crises](https://www.microsoft.com/en-us/research/wp-content/uploads/2009/07/hiLighter.pdf), by Peter Bodík, Moises Goldszmidt, Armando Fox, Hans Andersen. [**Microsoft**] |
+| Decision Tree | [**ICAC'04**] [Failure Diagnosis Using Decision Trees](http://www.cs.berkeley.edu/~brewer/papers/icac2004_chen_diagnosis.pdf), by Mike Chen, Alice X. Zheng, Jim Lloyd, Michael I. Jordan, Eric Brewer. [**eBay**] |
+| SVM | [**ICDM'07**] [Failure Prediction in IBM BlueGene/L Event Logs](https://www.researchgate.net/publication/4324148_Failure_Prediction_in_IBM_BlueGeneL_Event_Logs), by Yinglung Liang, Yanyong Zhang, Hui Xiong, Ramendra Sahoo. [**IBM**]|
+| **Unsupervised models** |
+| LOF | [**SIGMOD'00**] [LOF: Identifying Density-Based Local Outliers](), by Markus M. Breunig, Hans-Peter Kriegel, Raymond T. Ng, Jörg Sander. |
+| One-Class SVM | [**Neural Computation'01**] [Estimating the Support of a High-Dimensional Distribution](), by John Platt, Bernhard Schölkopf, John Shawe-Taylor, Alex J. Smola, Robert C. Williamson. |
+| Isolation Forest | [**ICDM'08**] [Isolation Forest](https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf), by Fei Tony Liu, Kai Ming Ting, Zhi-Hua Zhou. |
+| PCA | [**SOSP'09**] [Large-Scale System Problems Detection by Mining Console Logs](http://iiis.tsinghua.edu.cn/~weixu/files/sosp09.pdf), by Wei Xu, Ling Huang, Armando Fox, David Patterson, Michael I. Jordan. [**Intel**] |
+| Invariants Mining | [**ATC'10**] [Mining Invariants from Console Logs for System Problem Detection](https://www.usenix.org/legacy/event/atc10/tech/full_papers/Lou.pdf), by Jian-Guang Lou, Qiang Fu, Shengqi Yang, Ye Xu, Jiang Li. [**Microsoft**]|
+| Clustering | [**ICSE'16**] [Log Clustering based Problem Identification for Online Service Systems](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/07/ICSE-2016-2-Log-Clustering-based-Problem-Identification-for-Online-Service-Systems.pdf), by Qingwei Lin, Hongyu Zhang, Jian-Guang Lou, Yu Zhang, Xuewei Chen. [**Microsoft**]|
+| DeepLog (coming)| [**CCS'17**] [DeepLog: Anomaly Detection and Diagnosis from System Logs through Deep Learning](https://www.cs.utah.edu/~lifeifei/papers/deeplog.pdf), by Min Du, Feifei Li, Guineng Zheng, Vivek Srikumar. |
+| AutoEncoder (coming)| [**Arxiv'18**] [Anomaly Detection using Autoencoders in High Performance Computing Systems](https://arxiv.org/abs/1811.05269), by Andrea Borghesi, Andrea Bartolini, Michele Lombardi, Michela Milano, Luca Benini. |
+
+
+## Log data
+We have collected a set of labeled log datasets in [loghub](https://github.com/logpai/loghub) for research purposes. If you are interested in the datasets, please follow the link to submit your access request.
+
+## Install
+```bash
+git clone https://github.com/logpai/loglizer.git
+cd loglizer
+pip install -r requirements.txt
+```
+
+## API usage
+
+```python
+# Load HDFS dataset. If you would like to try your own log, you need to rewrite the load function.
+(x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(...)
+
+# Feature extraction and transformation
+feature_extractor = preprocessing.FeatureExtractor()
+feature_extractor.fit_transform(...) 
+
+# Model training
+model = PCA()
+model.fit(...)
+
+# Feature transform after fitting
+x_test = feature_extractor.transform(...)
+# Model evaluation with labeled data
+model.evaluate(...)
+
+# Anomaly prediction
+x_test = feature_extractor.transform(...)
+model.predict(...) # predict anomalies on given data
+```
+
+For more details, please follow [the demo](./docs/demo.md) in the docs to get started. Please note that all ML models are not magic, you need to figure out how to tune the parameters in order to make them work on your own data. 
+
+## Benchmarking results 
+
+If you would like to reproduce the following results, please run [benchmarks/HDFS_bechmark.py](./benchmarks/HDFS_bechmark.py) on the full HDFS dataset (HDFS100k is for demo only).
+
+|       |            | HDFS |     |
+| :----:|:----:|:----:|:----:|
+| **Model** | **Precision** | **Recall** | **F1** |
+| LR| 0.955 |	0.911 |	0.933 |
+| Decision Tree | 0.998 |	0.998 |	0.998 |
+| SVM| 0.959 |	0.970 |	0.965 |
+| LOF | 0.967 | 0.561 | 0.710 |
+| One-Class SVM | 0.995 | 0.222| 0.363 |
+| Isolation Forest |  0.830 | 0.776 | 0.802 |
+| PCA | 0.975 | 0.635 | 0.769|
+| Invariants Mining | 0.888 | 0.945 | 0.915|
+| Clustering | 1.000 | 0.720 | 0.837 |
+
+## Contributors
++ [Shilin He](https://shilinhe.github.io), The Chinese University of Hong Kong
++ [Jieming Zhu](https://jiemingzhu.github.io), The Chinese University of Hong Kong, currently at Huawei Noah's Ark Lab
++ [Pinjia He](https://pinjiahe.github.io/), The Chinese University of Hong Kong, currently at ETH Zurich
+
+
+## Feedback
+For any questions or feedback, please post to [the issue page](https://github.com/logpai/loglizer/issues/new). 
+
+"""
+#######################################################################################################################
+default_log_path = os.path.join("data","HDFS","HDFS_100k.log_structured.csv")
+def load_loglizer(train_log_path=default_log_path):
+    print("Loading Loglizer PCA model:")
+    start = time.time()
+    (x_train, _), (_, _), _ = dataloader.load_HDFS(train_log_path, window='session',
+                                                split_type='sequential', save_csv=True)
+    feature_extractor = preprocessing.FeatureExtractor()
+    x_train = feature_extractor.fit_transform(x_train, term_weighting='tf-idf',
+                                              normalization='zero-mean')
+    model = PCA()
+    model.fit(x_train)
+
+    stop = time.time()
+    print("Loading complete. Time elapsed: " + str(stop - start))
+
+    # Extract the event labels for each unique eventid in the source structured log file
+    # Load the CSV file
+    df = pd.read_csv(train_log_path)
+
+    # Extract unique 'EventId' and 'EventTemplate' pairs
+    event_names = df[['EventId', 'EventTemplate']].drop_duplicates()
+    event_names_dict = dict(zip(event_names['EventId'], event_names['EventTemplate']))
+
+    return model, feature_extractor, event_names_dict
+
+def analyze_with_chatgpt(anomalous_packets):
+    context = {"role":"system",
+               "content":"""
+               You will receive 5 rows of transformed data logs, where the middlemost log is flagged as anomalous.
+               Your job is to analyze and compare this log with its surrounding context logs, and find out why it was flagged as anomalous. 
+               Your response should be 50 to 150 words only; be as concise as possible and address the user directly. 
+               Only mention the key issues and cause of the anomaly. Never mention the Column Names, the middlemost log, surrounding context log.
+               You can mention what the irregular values in a column represent, and base your report on that.
+               Assume that the user has no knowledge of networks or cybersecurity. Your response should start with
+                "Based on the logs,".
+               Try to avoid telling the user 'if the issue persists'.
+               Give simple step-by-step advice on what the user can do on their own.
+               If the advice is beyond the scope of the everyday user, notify them that the assistance of a professional 
+               is necessary for the level of intrusion you found.
+               """}
+    message_prompt = []
+    message_prompt.append(context)
+    anomalies = f"*Column Names*: {anomalous_packets.columns} --- "
+    # Build a sequence of messages, with each message being a single packet up to the last packet flagged as anomalous
+    for i,p in anomalous_packets.iterrows():
+        anomalies += f"*Row {i+1}*: "
+        anomalies += str(p.values)
+        anomalies += " --- "
+
+    message_format = {"role":"user",
+                      "content":anomalies}
+    message_prompt.append(message_format)
+
+    # Generate the response
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=message_prompt,
+        max_tokens=1008,
+        temperature=0.7,
+        n=1,
+        stop=None,
+    )
+
+    # Extract the response text from the API response
+    response_text = response['choices'][0]['message']['content']
+    # Return the response text and updated chat history
+    return response_text
+
+def main():
+    head_col = st.columns([1,8])
+    with head_col[0]:
+        st.image(project_icon)
+    with head_col[1]:
+        st.title(project_title)
+
+    st.write(project_desc)
+    st.write(f"Source Project: {project_link}")
+    expander = st.expander("Additional Information on the Source Project (Loglizer)")
+    expander.markdown(add_info_md)
+    st.markdown("***")
+    st.subheader("")
+#########################################
+
+    # instructions and file upload button
+    st.subheader("""
+    How to use: 
+     1. Upload your log file (must be .csv format)
+     2. Click the 'Run Loglizer' button
+                 """)
+    uploaded_file = st.file_uploader("Upload a log file in csv format:",
+                               type=["csv"],
+                               accept_multiple_files=False)
+
+#########################################
+
+    run_button = st.button("Run Loglizer")
+
+    if "anomalies" not in st.session_state:
+        st.session_state.anomalies = []
+    if "col_names" not in st.session_state:
+        st.session_state.col_names = []
+
+    # button is clicked
+    if run_button:
+        if uploaded_file is None: # if no files were uploaded:
+            st.error("Please upload a .csv log file")
+
+        else:
+            # Ensure temp directory exists
+            if not os.path.exists('temp'):
+                os.makedirs('temp')
+
+            filename = os.path.basename(uploaded_file.name)
+            file_path = os.path.join("temp", filename)
+            # Write out the uploaded file to temp directory
+            with open(file_path, 'wb') as f:
+                f.write(uploaded_file.getbuffer())
+
+            with st.spinner('Loading Loglizer...'):
+                model, feature_extractor, event_names_dict = load_loglizer()
+
+            # Simulate the 'live' log feed
+            (x_live, _), (_, _), _ = dataloader.load_HDFS(file_path, window='session', split_type='sequential')
+            x_live, st.session_state.col_names = feature_extractor.transform(x_live)
+            # st.write(f"x_live shape: {x_live.shape}")
+
+            # Map event IDs to event templates
+            st.session_state.col_names = [event_names_dict[id] if id in event_names_dict else id for id in
+                                          st.session_state.col_names]
+
+            for i in range(len(x_live)):
+                log = x_live[i]
+                log = np.expand_dims(log, axis=0)
+                prediction = model.predict(log)
+                print(f"prediction for x_live[{i}]: {prediction}")
+                if prediction == 1:
+                    # Anomaly detected
+                    context = x_live[max(0, i - 2):min(i + 3, len(x_live))]  # Get the two logs before and after
+                    st.session_state.anomalies.append(context)
+            # st.write("predict on x_live:")
+            # st.write(model.predict(x_live))
+
+    st.write("Anomalous events")
+    st.write(len(st.session_state.anomalies))
+    #
+    # st.write("Column names")
+    # st.write(st.session_state.col_names)
+
+    if len(st.session_state.anomalies) > 0:
+        # Displaying the anomalous packets and sending to ChatGPT
+        selected_index = st.selectbox('Select an anomaly', list(range(len(st.session_state.anomalies))), 0)
+        selected_anomaly = st.session_state.anomalies[selected_index]
+        st.write(f"Loglizer found and compiled {selected_anomaly.shape[1]} events from the logs.\n The middle entry has an anomaly:")
+        # st.write(f"Shape: {selected_anomaly.shape} Type: {type(selected_anomaly)}")
+
+        # Convert the numpy array to a pandas DataFrame
+        selected_anomaly_df = pd.DataFrame(selected_anomaly)
+        # Set the column names
+        selected_anomaly_df.columns = st.session_state.col_names
+
+        # st.write(selected_anomaly)
+        st.write(selected_anomaly_df)
+
+        if st.button('Send to ChatGPT'):
+            result = analyze_with_chatgpt(selected_anomaly_df)
+            st.write(result)
+
+
+if __name__ == '__main__':
+    main()
+
+# To run streamlit, go to terminal and type: 'streamlit run app-source.py'

data/data_instances.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba4d353dddd4e7ef0de168def9502f38d17dbe95ca5d966c5a6d68aa05b4955
+size 909112

data/loglizer/dataloader.py

@@ -0,0 +1,268 @@
+"""
+The interface to load log datasets. The datasets currently supported include
+HDFS and BGL.
+
+Authors:
+    LogPAI Team
+
+"""
+
+import pandas as pd
+import os
+import numpy as np
+import re
+from sklearn.utils import shuffle
+from collections import OrderedDict
+
+def _split_data(x_data, y_data=None, train_ratio=0, split_type='uniform'):
+    if split_type == 'uniform' and y_data is not None:
+        pos_idx = y_data > 0
+        x_pos = x_data[pos_idx]
+        y_pos = y_data[pos_idx]
+        x_neg = x_data[~pos_idx]
+        y_neg = y_data[~pos_idx]
+        train_pos = int(train_ratio * x_pos.shape[0])
+        train_neg = int(train_ratio * x_neg.shape[0])
+        x_train = np.hstack([x_pos[0:train_pos], x_neg[0:train_neg]])
+        y_train = np.hstack([y_pos[0:train_pos], y_neg[0:train_neg]])
+        x_test = np.hstack([x_pos[train_pos:], x_neg[train_neg:]])
+        y_test = np.hstack([y_pos[train_pos:], y_neg[train_neg:]])
+    elif split_type == 'sequential':
+        num_train = int(train_ratio * x_data.shape[0])
+        x_train = x_data[0:num_train]
+        x_test = x_data[num_train:]
+        if y_data is None:
+            y_train = None
+            y_test = None
+        else:
+            y_train = y_data[0:num_train]
+            y_test = y_data[num_train:]
+    # Random shuffle
+    indexes = shuffle(np.arange(x_train.shape[0]))
+    x_train = x_train[indexes]
+    if y_train is not None:
+        y_train = y_train[indexes]
+    return (x_train, y_train), (x_test, y_test)
+
+def load_HDFS(log_file, label_file=None, window='session', train_ratio=0.5, split_type='sequential', save_csv=False, window_size=0):
+    """ Load HDFS structured log into train and test data
+
+    Arguments
+    ---------
+        log_file: str, the file path of structured log.
+        label_file: str, the file path of anomaly labels, None for unlabeled data
+        window: str, the window options including `session` (default).
+        train_ratio: float, the ratio of training data for train/test split.
+        split_type: `uniform` or `sequential`, which determines how to split dataset. `uniform` means
+            to split positive samples and negative samples equally when setting label_file. `sequential`
+            means to split the data sequentially without label_file. That is, the first part is for training,
+            while the second part is for testing.
+
+    Returns
+    -------
+        (x_train, y_train): the training data
+        (x_test, y_test): the testing data
+    """
+
+    print('====== Input data summary ======')
+
+    if log_file.endswith('.npz'):
+        # Split training and validation set in a class-uniform way
+        data = np.load(log_file)
+        x_data = data['x_data']
+        y_data = data['y_data']
+        (x_train, y_train), (x_test, y_test) = _split_data(x_data, y_data, train_ratio, split_type)
+
+    elif log_file.endswith('.csv'):
+        assert window == 'session', "Only window=session is supported for HDFS dataset."
+        print("Loading", log_file)
+        struct_log = pd.read_csv(log_file, engine='c',
+                na_filter=False, memory_map=True)
+        data_dict = OrderedDict()
+        for idx, row in struct_log.iterrows():
+            blkId_list = re.findall(r'(blk_-?\d+)', row['Content'])
+            blkId_set = set(blkId_list)
+            for blk_Id in blkId_set:
+                if not blk_Id in data_dict:
+                    data_dict[blk_Id] = []
+                data_dict[blk_Id].append(row['EventId'])
+        data_df = pd.DataFrame(list(data_dict.items()), columns=['BlockId', 'EventSequence'])
+        
+        if label_file:
+            # Split training and validation set in a class-uniform way
+            label_data = pd.read_csv(label_file, engine='c', na_filter=False, memory_map=True)
+            label_data = label_data.set_index('BlockId')
+            label_dict = label_data['Label'].to_dict()
+            data_df['Label'] = data_df['BlockId'].apply(lambda x: 1 if label_dict[x] == 'Anomaly' else 0)
+
+            # Split train and test data
+            (x_train, y_train), (x_test, y_test) = _split_data(data_df['EventSequence'].values, 
+                data_df['Label'].values, train_ratio, split_type)
+        
+            print(y_train.sum(), y_test.sum())
+
+        if save_csv:
+            data_df.to_csv('data_instances.csv', index=False)
+
+        if window_size > 0:
+            x_train, window_y_train, y_train = slice_hdfs(x_train, y_train, window_size)
+            x_test, window_y_test, y_test = slice_hdfs(x_test, y_test, window_size)
+            log = "{} {} windows ({}/{} anomaly), {}/{} normal"
+            print(log.format("Train:", x_train.shape[0], y_train.sum(), y_train.shape[0], (1-y_train).sum(), y_train.shape[0]))
+            print(log.format("Test:", x_test.shape[0], y_test.sum(), y_test.shape[0], (1-y_test).sum(), y_test.shape[0]))
+            return (x_train, window_y_train, y_train), (x_test, window_y_test, y_test)
+
+        if label_file is None:
+            if split_type == 'uniform':
+                split_type = 'sequential'
+                print('Warning: Only split_type=sequential is supported \
+                if label_file=None.'.format(split_type))
+            # Split training and validation set sequentially
+            x_data = data_df['EventSequence'].values
+            (x_train, _), (x_test, _) = _split_data(x_data, train_ratio=train_ratio, split_type=split_type)
+            print('Total: {} instances, train: {} instances, test: {} instances'.format(
+                  x_data.shape[0], x_train.shape[0], x_test.shape[0]))
+            return (x_train, None), (x_test, None), data_df
+    else:
+        raise NotImplementedError('load_HDFS() only support csv and npz files!')
+
+    num_train = x_train.shape[0]
+    num_test = x_test.shape[0]
+    num_total = num_train + num_test
+    num_train_pos = sum(y_train)
+    num_test_pos = sum(y_test)
+    num_pos = num_train_pos + num_test_pos
+
+    print('Total: {} instances, {} anomaly, {} normal' \
+          .format(num_total, num_pos, num_total - num_pos))
+    print('Train: {} instances, {} anomaly, {} normal' \
+          .format(num_train, num_train_pos, num_train - num_train_pos))
+    print('Test: {} instances, {} anomaly, {} normal\n' \
+          .format(num_test, num_test_pos, num_test - num_test_pos))
+
+    return (x_train, y_train), (x_test, y_test)
+
+def slice_hdfs(x, y, window_size):
+    results_data = []
+    print("Slicing {} sessions, with window {}".format(x.shape[0], window_size))
+    for idx, sequence in enumerate(x):
+        seqlen = len(sequence)
+        i = 0
+        while (i + window_size) < seqlen:
+            slice = sequence[i: i + window_size]
+            results_data.append([idx, slice, sequence[i + window_size], y[idx]])
+            i += 1
+        else:
+            slice = sequence[i: i + window_size]
+            slice += ["#Pad"] * (window_size - len(slice))
+            results_data.append([idx, slice, "#Pad", y[idx]])
+    results_df = pd.DataFrame(results_data, columns=["SessionId", "EventSequence", "Label", "SessionLabel"])
+    print("Slicing done, {} windows generated".format(results_df.shape[0]))
+    return results_df[["SessionId", "EventSequence"]], results_df["Label"], results_df["SessionLabel"]
+
+
+
+def load_BGL(log_file, label_file=None, window='sliding', time_interval=60, stepping_size=60, 
+             train_ratio=0.8):
+    """  TODO
+
+    """
+
+
+def bgl_preprocess_data(para, raw_data, event_mapping_data):
+    """ split logs into sliding windows, built an event count matrix and get the corresponding label
+
+    Args:
+    --------
+    para: the parameters dictionary
+    raw_data: list of (label, time)
+    event_mapping_data: a list of event index, where each row index indicates a corresponding log
+
+    Returns:
+    --------
+    event_count_matrix: event count matrix, where each row is an instance (log sequence vector)
+    labels: a list of labels, 1 represents anomaly
+    """
+
+    # create the directory for saving the sliding windows (start_index, end_index), which can be directly loaded in future running
+    if not os.path.exists(para['save_path']):
+        os.mkdir(para['save_path'])
+    log_size = raw_data.shape[0]
+    sliding_file_path = para['save_path']+'sliding_'+str(para['window_size'])+'h_'+str(para['step_size'])+'h.csv'
+
+    #=============divide into sliding windows=========#
+    start_end_index_list = [] # list of tuples, tuple contains two number, which represent the start and end of sliding time window
+    label_data, time_data = raw_data[:,0], raw_data[:, 1]
+    if not os.path.exists(sliding_file_path):
+        # split into sliding window
+        start_time = time_data[0]
+        start_index = 0
+        end_index = 0
+
+        # get the first start, end index, end time
+        for cur_time in time_data:
+            if  cur_time < start_time + para['window_size']*3600:
+                end_index += 1
+                end_time = cur_time
+            else:
+                start_end_pair=tuple((start_index,end_index))
+                start_end_index_list.append(start_end_pair)
+                break
+        # move the start and end index until next sliding window
+        while end_index < log_size:
+            start_time = start_time + para['step_size']*3600
+            end_time = end_time + para['step_size']*3600
+            for i in range(start_index,end_index):
+                if time_data[i] < start_time:
+                    i+=1
+                else:
+                    break
+            for j in range(end_index, log_size):
+                if time_data[j] < end_time:
+                    j+=1
+                else:
+                    break
+            start_index = i
+            end_index = j
+            start_end_pair = tuple((start_index, end_index))
+            start_end_index_list.append(start_end_pair)
+        inst_number = len(start_end_index_list)
+        print('there are %d instances (sliding windows) in this dataset\n'%inst_number)
+        np.savetxt(sliding_file_path,start_end_index_list,delimiter=',',fmt='%d')
+    else:
+        print('Loading start_end_index_list from file')
+        start_end_index_list = pd.read_csv(sliding_file_path, header=None).values
+        inst_number = len(start_end_index_list)
+        print('there are %d instances (sliding windows) in this dataset' % inst_number)
+
+    # get all the log indexes in each time window by ranging from start_index to end_index
+    expanded_indexes_list=[]
+    for t in range(inst_number):
+        index_list = []
+        expanded_indexes_list.append(index_list)
+    for i in range(inst_number):
+        start_index = start_end_index_list[i][0]
+        end_index = start_end_index_list[i][1]
+        for l in range(start_index, end_index):
+            expanded_indexes_list[i].append(l)
+
+    event_mapping_data = [row[0] for row in event_mapping_data]
+    event_num = len(list(set(event_mapping_data)))
+    print('There are %d log events'%event_num)
+
+    #=============get labels and event count of each sliding window =========#
+    labels = []
+    event_count_matrix = np.zeros((inst_number,event_num))
+    for j in range(inst_number):
+        label = 0   #0 represent success, 1 represent failure
+        for k in expanded_indexes_list[j]:
+            event_index = event_mapping_data[k]
+            event_count_matrix[j, event_index] += 1
+            if label_data[k]:
+                label = 1
+                continue
+        labels.append(label)
+    assert inst_number == len(labels)
+    print("Among all instances, %d are anomalies"%sum(labels))
+    assert event_count_matrix.shape[0] == len(labels)
+    return event_count_matrix, labels

data/loglizer/models/DecisionTree.py

@@ -0,0 +1,78 @@
+"""
+The implementation of the decision tree model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Mike Chen, Alice X. Zheng, Jim Lloyd, Michael I. Jordan, Eric Brewer. 
+        Failure Diagnosis Using Decision Trees. IEEE International Conference 
+        on Autonomic Computing (ICAC), 2004.
+
+"""
+
+import numpy as np
+from sklearn import tree
+from ..utils import metrics
+
+class DecisionTree(object):
+
+    def __init__(self, criterion='gini', max_depth=None, max_features=None, class_weight=None):
+        """ The Invariants Mining model for anomaly detection
+        Arguments
+        ---------
+        See DecisionTreeClassifier API: https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html
+
+        Attributes
+        ----------
+            classifier: object, the classifier for anomaly detection
+
+        """
+        self.classifier = tree.DecisionTreeClassifier(criterion=criterion, max_depth=max_depth,
+                          max_features=max_features, class_weight=class_weight)
+
+    def fit(self, X, y):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        self.classifier.fit(X, y)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = self.classifier.predict(X)
+        return y_pred
+
+    def predict_proba(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = self.classifier.predict_proba(X)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1

data/loglizer/models/InvariantsMiner.py

@@ -0,0 +1,296 @@
+"""
+The implementation of Invariants Mining model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Jian-Guang Lou, Qiang Fu, Shengqi Yang, Ye Xu, Jiang Li. Mining Invariants 
+        from Console Logs for System Problem Detection. USENIX Annual Technical 
+        Conference (ATC), 2010.
+
+"""
+
+import numpy as np
+from itertools import combinations
+from ..utils import metrics
+
+class InvariantsMiner(object):
+
+    def __init__(self, percentage=0.98, epsilon=0.5, longest_invarant=None, scale_list=[1,2,3]):
+        """ The Invariants Mining model for anomaly detection
+
+        Attributes
+        ----------
+            percentage: float, percentage of samples satisfying the condition that |X_j * V_i| < epsilon
+            epsilon: float, the threshold for estimating the invariant space
+            longest_invarant: int, the specified maximal length of invariant, default to None. Stop 
+                searching when the invariant length is larger than longest_invarant.
+            scale_list: list, the list used to scale the theta of float into integer
+            invariants_dict: dict, dictionary of invariants where key is the selected columns 
+                and value is the weights the of invariant
+        """
+        self.percentage = percentage
+        self.epsilon = epsilon
+        self.longest_invarant = longest_invarant
+        self.scale_list = scale_list
+        self.invariants_dict = None
+
+    def fit(self, X):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        invar_dim = self._estimate_invarant_space(X)
+        self._invariants_search(X, invar_dim)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_sum = np.zeros(X.shape[0])
+        for cols, theta in self.invariants_dict.items():
+            y_sum += np.fabs(np.dot(X[:, cols], np.array(theta)))
+        y_pred = (y_sum > 1e-6).astype(int)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1
+
+    def _estimate_invarant_space(self, X):
+        """ Estimate the dimension of invariant space using SVD decomposition
+
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+            percentage: float, percentage of samples satisfying the condition that |X_j * V_i| < epsilon
+            epsilon: float, the threshold for estimating the invariant space
+
+        Returns
+        -------
+            r: the dimension of invariant space
+        """
+        covariance_matrix = np.dot(X.T, X)
+        U, sigma, V = np.linalg.svd(covariance_matrix)  # SVD decomposition
+        # Start from the right most column of matrix V, sigular values are in ascending order
+        num_instances, num_events = X.shape
+        r = 0
+        for i in range(num_events - 1, -1, -1):
+            zero_count = sum(abs(np.dot(X, U[:, i])) < self.epsilon)
+            if zero_count / float(num_instances) < self.percentage:
+                break
+            r += 1
+        print('Invariant space dimension: {}'.format(r))
+
+        return r
+
+    def _invariants_search(self, X, r):
+        """ Mine invariant relationships from X
+
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+            r: the dimension of invariant space
+        """
+
+        num_instances, num_events = X.shape
+        invariants_dict = dict()  # save the mined Invariants(value) and its corresponding columns(key)
+        search_space = []  # only invariant candidates in this list are valid
+
+        # invariant of only one column (all zero columns)
+        init_cols = sorted([[item] for item in range(num_events)])
+        for col in init_cols:
+            search_space.append(col)
+        init_col_list = init_cols[:]
+        for col in init_cols:
+            if np.count_nonzero(X[:, col]) == 0:
+                invariants_dict[tuple(col)] = [1]
+                search_space.remove(col)
+                init_col_list.remove(col)
+
+        item_list = init_col_list
+        length = 2
+        FLAG_break_loop = False
+        # check invariant of more columns
+        while len(item_list) != 0:
+            if self.longest_invarant and len(item_list[0]) >= self.longest_invarant:
+                break
+            joined_item_list = self._join_set(item_list, length) # generate new invariant candidates
+            for items in joined_item_list:
+                if self._check_candi_valid(items, length, search_space):
+                    search_space.append(items)
+            item_list = []
+            for item in joined_item_list:
+                if tuple(item) in invariants_dict:
+                    continue
+                if item not in search_space:
+                    continue
+                if not self._check_candi_valid(tuple(item), length, search_space) and length > 2:
+                    search_space.remove(item)
+                    continue # an item must be superset of all other subitems in searchSpace, else skip
+                validity, scaled_theta = self._check_invar_validity(X, item)
+                if validity:
+                    self._prune(invariants_dict.keys(), set(item), search_space)
+                    invariants_dict[tuple(item)] = scaled_theta.tolist()
+                    search_space.remove(item)
+                else:
+                    item_list.append(item)
+                if len(invariants_dict) >= r:
+                    FLAG_break_loop = True
+                    break
+            if FLAG_break_loop:
+                break
+            length += 1
+        print('Mined {} invariants: {}\n'.format(len(invariants_dict), invariants_dict))
+        self.invariants_dict = invariants_dict
+
+    def _compute_eigenvector(self, X):
+        """ calculate the smallest eigenvalue and corresponding eigenvector (theta in the paper) 
+            for a given sub_matrix
+
+        Arguments
+        ---------
+            X: the event count matrix (each row is a log sequence vector, each column represents an event)
+
+        Returns
+        -------
+            min_vec: the eigenvector of corresponding minimum eigen value
+            FLAG_contain_zero: whether the min_vec contains zero (very small value)
+        """
+
+        FLAG_contain_zero = False
+        count_zero = 0
+        dot_result = np.dot(X.T, X)
+        U, S, V = np.linalg.svd(dot_result)
+        min_vec = U[:, -1]
+        count_zero = sum(np.fabs(min_vec) < 1e-6)
+        if count_zero != 0:
+            FLAG_contain_zero = True
+        return min_vec, FLAG_contain_zero
+
+
+    def _check_invar_validity(self, X, selected_columns):
+        """ scale the eigenvector of float number into integer, and check whether the scaled number is valid
+
+        Arguments
+        ---------
+            X: the event count matrix (each row is a log sequence vector, each column represents an event)
+            selected_columns: select columns from all column list
+
+        Returns
+        -------
+            validity: whether the selected columns is valid
+            scaled_theta: the scaled theta vector
+        """
+
+        sub_matrix = X[:, selected_columns]
+        inst_num = X.shape[0]
+        validity = False
+        min_theta, FLAG_contain_zero = self._compute_eigenvector(sub_matrix)
+        abs_min_theta = [np.fabs(it) for it in min_theta]
+        if FLAG_contain_zero:
+            return validity, []
+        else:
+            for i in self.scale_list:
+                min_index = np.argmin(abs_min_theta)
+                scale = float(i) / min_theta[min_index]
+                scaled_theta = np.array([round(item * scale) for item in min_theta])
+                scaled_theta[min_index] = i
+                scaled_theta = scaled_theta.T
+                if 0 in np.fabs(scaled_theta):
+                    continue
+                dot_submat_theta = np.dot(sub_matrix, scaled_theta)
+                count_zero = 0
+                for j in dot_submat_theta:
+                    if np.fabs(j) < 1e-8:
+                        count_zero += 1
+                if count_zero >= self.percentage * inst_num:
+                    validity = True
+                    # print('A valid invariant is found: ',scaled_theta, selected_columns)
+                    break
+            return validity, scaled_theta
+
+    def _prune(self, valid_cols, new_item_set, search_space):
+        """ prune invalid combination of columns
+
+        Arguments
+        ---------
+            valid_cols: existing valid column list
+            new_item_set: item set to be merged
+            search_space: the search space that stores possible candidates
+
+        """
+
+        if len(valid_cols) == 0:
+            return
+        for se in valid_cols:
+            intersection = set(se) & new_item_set
+            if len(intersection) == 0:
+                continue
+            union = set(se) | new_item_set
+            for item in list(intersection):
+                diff = sorted(list(union - set([item])))
+                if diff in search_space:
+                    search_space.remove(diff)
+
+
+    def _join_set(self, item_list, length):
+        """ Join a set with itself and returns the n-element (length) itemsets
+
+        Arguments
+        ---------
+            item_list: current list of columns
+            length: generate new items of length
+
+        Returns
+        -------
+            return_list: list of items of length-element
+        """
+
+        set_len = len(item_list)
+        return_list = []
+        for i in range(set_len):
+            for j in range(i + 1, set_len):
+                i_set = set(item_list[i])
+                j_set = set(item_list[j])
+                if len(i_set.union(j_set)) == length:
+                    joined = sorted(list(i_set.union(j_set)))
+                    if joined not in return_list:
+                        return_list.append(joined)
+        return_list = sorted(return_list)
+        return return_list
+
+
+    def _check_candi_valid(self, item, length, search_space):
+        """ check whether an item's subitems are in searchspace
+
+        Arguments
+        ---------
+            item: item to be checked
+            length: the length of item
+            search_space: the search space that stores possible candidates
+
+        Returns
+        -------
+            True or False
+        """
+
+        for subItem in combinations(item, length - 1):
+            if sorted(list(subItem)) not in search_space:
+                return False
+        return True
+

data/loglizer/models/IsolationForest.py

@@ -0,0 +1,96 @@
+"""
+The implementation of IsolationForest model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Fei Tony Liu, Kai Ming Ting, Zhi-Hua Zhou. Isolation Forest. International
+        Conference on Data Mining (ICDM), 2008.
+
+"""
+
+
+
+
+
+import numpy as np
+from sklearn.ensemble import IsolationForest as iForest
+from ..utils import metrics
+
+class IsolationForest(iForest):
+
+    def __init__(self, n_estimators=100, max_samples='auto', contamination=0.03, **kwargs):
+        """ The IsolationForest model for anomaly detection
+
+        Arguments
+        ---------
+            n_estimators : int, optional (default=100). The number of base estimators in the ensemble.
+            max_samples : int or float, optional (default="auto")
+                The number of samples to draw from X to train each base estimator.
+                    - If int, then draw max_samples samples.
+                    - If float, then draw max_samples * X.shape[0] samples.
+                    - If "auto", then max_samples=min(256, n_samples).
+                If max_samples is larger than the number of samples provided, all samples will be used 
+                for all trees (no sampling).
+            contamination : float in (0., 0.5), optional (default='auto')
+                The amount of contamination of the data set, i.e. the proportion of outliers in the data 
+                set. Used when fitting to define the threshold on the decision function. If 'auto', the 
+                decision function threshold is determined as in the original paper.
+            max_features : int or float, optional (default=1.0)
+                The number of features to draw from X to train each base estimator.
+                    - If int, then draw max_features features.
+                    - If float, then draw max_features * X.shape[1] features.
+            bootstrap : boolean, optional (default=False)
+                If True, individual trees are fit on random subsets of the training data sampled with replacement. 
+                If False, sampling without replacement is performed.
+            n_jobs : int or None, optional (default=None)
+                The number of jobs to run in parallel for both fit and predict. None means 1 unless in a 
+                joblib.parallel_backend context. -1 means using all processors. 
+            random_state : int, RandomState instance or None, optional (default=None)
+                If int, random_state is the seed used by the random number generator; 
+                If RandomState instance, random_state is the random number generator; 
+                If None, the random number generator is the RandomState instance used by np.random.
+        
+        Reference
+        ---------
+            For more information, please visit https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html
+        """
+
+        super(IsolationForest, self).__init__(n_estimators=n_estimators, max_samples=max_samples, 
+            contamination=contamination, **kwargs)
+
+
+    def fit(self, X):
+        """
+        Auguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+
+        print('====== Model summary ======')
+        super(IsolationForest, self).fit(X)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = super(IsolationForest, self).predict(X)
+        y_pred = np.where(y_pred > 0, 0, 1)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1
+

data/loglizer/models/LR.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+The implementation of the logistic regression model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Peter Bodík, Moises Goldszmidt, Armando Fox, Hans Andersen. Fingerprinting 
+        the Datacenter: Automated Classification of Performance Crises. The European 
+        Conference on Computer Systems (EuroSys), 2010.
+
+"""
+
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from ..utils import metrics
+
+class LR(object):
+
+    def __init__(self, penalty='l2', C=100, tol=0.01, class_weight=None, max_iter=100):
+        """ The Invariants Mining model for anomaly detection
+
+        Attributes
+        ----------
+            classifier: object, the classifier for anomaly detection
+        """
+        self.classifier = LogisticRegression(penalty=penalty, C=C, tol=tol, class_weight=class_weight,
+                                             max_iter=max_iter)
+
+    def fit(self, X, y):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        self.classifier.fit(X, y)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        y_pred = self.classifier.predict(X)
+        return y_pred
+
+    def predict_proba(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        y_pred = self.classifier.predict_proba(X)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1

data/loglizer/models/LogClustering.py

@@ -0,0 +1,137 @@
+"""
+The implementation of Log Clustering model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Qingwei Lin, Hongyu Zhang, Jian-Guang Lou, Yu Zhang, Xuewei Chen. Log Clustering 
+        based Problem Identification for Online Service Systems. International Conference
+        on Software Engineering (ICSE), 2016.
+
+"""
+
+import numpy as np
+import pprint
+from scipy.special import expit
+from numpy import linalg as LA
+from scipy.cluster.hierarchy import linkage, fcluster
+from scipy.spatial.distance import pdist, squareform
+from ..utils import metrics
+
+
+class LogClustering(object):
+
+    def __init__(self, max_dist=0.3, anomaly_threshold=0.3, mode='online', num_bootstrap_samples=1000):
+        """
+        Attributes
+        ----------
+            max_dist: float, the threshold to stop the clustering process
+            anomaly_threshold: float, the threshold for anomaly detection
+            mode: str, 'offline' or 'online' mode for clustering
+            num_bootstrap_samples: int, online clustering starts with a bootstraping process, which
+                determines the initial cluster representatives offline using a subset of samples 
+            representatives: ndarray, the representative samples of clusters, of shape 
+                num_clusters-by-num_events
+            cluster_size_dict: dict, the size of each cluster, used to update representatives online 
+        """
+        self.max_dist = max_dist
+        self.anomaly_threshold = anomaly_threshold
+        self.mode = mode
+        self.num_bootstrap_samples = num_bootstrap_samples
+        self.representatives = list()
+        self.cluster_size_dict = dict()
+
+    def fit(self, X):   
+        print('====== Model summary ======')         
+        if self.mode == 'offline':
+            # The offline mode can process about 10K samples only due to huge memory consumption.
+            self._offline_clustering(X)
+        elif self.mode == 'online':
+            # Bootstrapping phase
+            if self.num_bootstrap_samples > 0:
+                X_bootstrap = X[0:self.num_bootstrap_samples, :]
+                self._offline_clustering(X_bootstrap)
+            # Online learning phase
+            if X.shape[0] > self.num_bootstrap_samples:
+                self._online_clustering(X)
+
+    def predict(self, X):
+        y_pred = np.zeros(X.shape[0])
+        for i in range(X.shape[0]):
+            min_dist, min_index = self._get_min_cluster_dist(X[i, :])
+            if min_dist > self.anomaly_threshold:
+                y_pred[i] = 1
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n' \
+              .format(precision, recall, f1))
+        return precision, recall, f1
+
+    def _offline_clustering(self, X):
+        print('Starting offline clustering...')
+        p_dist = pdist(X, metric=self._distance_metric)
+        Z = linkage(p_dist, 'complete')
+        cluster_index = fcluster(Z, self.max_dist, criterion='distance')
+        self._extract_representatives(X, cluster_index)
+        print('Processed {} instances.'.format(X.shape[0]))
+        print('Found {} clusters offline.\n'.format(len(self.representatives)))
+        # print('The representive vectors are:')
+        # pprint.pprint(self.representatives.tolist())
+
+    def _extract_representatives(self, X, cluster_index):
+        num_clusters = len(set(cluster_index))
+        for clu in range(num_clusters):
+            clu_idx = np.argwhere(cluster_index == clu + 1)[:, 0]
+            self.cluster_size_dict[clu] = clu_idx.shape[0]
+            repre_center = np.average(X[clu_idx, :], axis=0)
+            self.representatives.append(repre_center)
+
+    def _online_clustering(self, X):
+        print("Starting online clustering...")
+        for i in range(self.num_bootstrap_samples, X.shape[0]):
+            if (i + 1) % 2000 == 0:
+                print('Processed {} instances.'.format(i + 1))
+            instance_vec = X[i, :]
+            if len(self.representatives) > 0:
+                min_dist, clu_id = self._get_min_cluster_dist(instance_vec)
+                if min_dist <= self.max_dist:
+                    self.cluster_size_dict[clu_id] += 1
+                    self.representatives[clu_id] = self.representatives[clu_id] \
+                                                 + (instance_vec - self.representatives[clu_id]) \
+                                                 / self.cluster_size_dict[clu_id]
+                    continue
+            self.cluster_size_dict[len(self.representatives)] = 1
+            self.representatives.append(instance_vec)
+        print('Processed {} instances.'.format(X.shape[0]))
+        print('Found {} clusters online.\n'.format(len(self.representatives)))
+        # print('The representive vectors are:')
+        # pprint.pprint(self.representatives.tolist())
+
+    def _distance_metric(self, x1, x2):
+        norm= LA.norm(x1) * LA.norm(x2)
+        distance = 1 - np.dot(x1, x2) / (norm + 1e-8)
+        if distance < 1e-8:
+            distance = 0
+        return distance
+
+    def _get_min_cluster_dist(self, instance_vec):
+        min_index = -1
+        min_dist = float('inf')
+        for i in range(len(self.representatives)):
+            cluster_rep = self.representatives[i]
+            dist = self._distance_metric(instance_vec, cluster_rep)
+            if dist < 1e-8:
+                min_dist = 0
+                min_index = i
+                break
+            elif dist < min_dist:
+                min_dist = dist
+                min_index = i
+        return min_dist, min_index
+
+

data/loglizer/models/PCA.py

@@ -0,0 +1,105 @@
+"""
+The implementation of PCA model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Wei Xu, Ling Huang, Armando Fox, David Patterson, Michael I. Jordan. 
+        Large-Scale System Problems Detection by Mining Console Logs. ACM 
+        Symposium on Operating Systems Principles (SOSP), 2009.
+
+"""
+
+import numpy as np
+from ..utils import metrics
+
+class PCA(object):
+
+    def __init__(self, n_components=0.95, threshold=None, c_alpha=3.2905):
+        """ The PCA model for anomaly detection
+
+        Attributes
+        ----------
+            proj_C: The projection matrix for projecting feature vector to abnormal space
+            n_components: float/int, number of principal compnents or the variance ratio they cover
+            threshold: float, the anomaly detection threshold. When setting to None, the threshold 
+                is automatically caculated using Q-statistics
+            c_alpha: float, the c_alpha parameter for caculating anomaly detection threshold using 
+                Q-statistics. The following is lookup table for c_alpha:
+                c_alpha = 1.7507; # alpha = 0.08
+                c_alpha = 1.9600; # alpha = 0.05
+                c_alpha = 2.5758; # alpha = 0.01
+                c_alpha = 2.807; # alpha = 0.005
+                c_alpha = 2.9677;  # alpha = 0.003
+                c_alpha = 3.2905;  # alpha = 0.001
+                c_alpha = 3.4808;  # alpha = 0.0005
+                c_alpha = 3.8906;  # alpha = 0.0001
+                c_alpha = 4.4172;  # alpha = 0.00001
+        """
+
+        self.proj_C = None
+        self.components = None
+        self.n_components = n_components
+        self.threshold = threshold
+        self.c_alpha = c_alpha
+
+
+    def fit(self, X):
+        """
+        Auguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+
+        print('====== Model summary ======')
+        num_instances, num_events = X.shape
+        X_cov = np.dot(X.T, X) / float(num_instances)
+        U, sigma, V = np.linalg.svd(X_cov)
+        n_components = self.n_components
+        if n_components < 1:
+            total_variance = np.sum(sigma)
+            variance = 0
+            for i in range(num_events):
+                variance += sigma[i]
+                if variance / total_variance >= n_components:
+                    break
+            n_components = i + 1
+
+        P = U[:, :n_components]
+        I = np.identity(num_events, int)
+        self.components = P
+        self.proj_C = I - np.dot(P, P.T)
+        print('n_components: {}'.format(n_components))
+        print('Project matrix shape: {}-by-{}'.format(self.proj_C.shape[0], self.proj_C.shape[1]))
+
+        if not self.threshold:
+            # Calculate threshold using Q-statistic. Information can be found at:
+            # http://conferences.sigcomm.org/sigcomm/2004/papers/p405-lakhina111.pdf
+            phi = np.zeros(3)
+            for i in range(3):
+                for j in range(n_components, num_events):
+                    phi[i] += np.power(sigma[j], i + 1)
+            h0 = 1.0 - 2 * phi[0] * phi[2] / (3.0 * phi[1] * phi[1])
+            self.threshold = phi[0] * np.power(self.c_alpha * np.sqrt(2 * phi[1] * h0 * h0) / phi[0]
+                                               + 1.0 + phi[1] * h0 * (h0 - 1) / (phi[0] * phi[0]), 
+                                               1.0 / h0)
+        print('SPE threshold: {}\n'.format(self.threshold))
+
+    def predict(self, X):
+        assert self.proj_C is not None, 'PCA model needs to be trained before prediction.'
+        y_pred = np.zeros(X.shape[0])
+        for i in range(X.shape[0]):
+            y_a = np.dot(self.proj_C, X[i, :])
+            SPE = np.dot(y_a, y_a)
+            if SPE > self.threshold:
+                y_pred[i] = 1
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1
+

data/loglizer/models/SVM.py

@@ -0,0 +1,64 @@
+"""
+The implementation of the SVM model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Yinglung Liang, Yanyong Zhang, Hui Xiong, Ramendra Sahoo. Failure Prediction 
+        in IBM BlueGene/L Event Logs. IEEE International Conference on Data Mining
+        (ICDM), 2007.
+
+"""
+
+import numpy as np
+from sklearn import svm
+from ..utils import metrics
+
+class SVM(object):
+
+    def __init__(self, penalty='l1', tol=0.1, C=1, dual=False, class_weight=None, 
+                 max_iter=100):
+        """ The Invariants Mining model for anomaly detection
+        Arguments
+        ---------
+        See SVM API: https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html
+        
+        Attributes
+        ----------
+            classifier: object, the classifier for anomaly detection
+
+        """
+        self.classifier = svm.LinearSVC(penalty=penalty, tol=tol, C=C, dual=dual, 
+                                        class_weight=class_weight, max_iter=max_iter)
+
+    def fit(self, X, y):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        self.classifier.fit(X, y)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = self.classifier.predict(X)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1

data/loglizer/models/__init__.py

@@ -0,0 +1,8 @@
+from .PCA import PCA
+from .InvariantsMiner import InvariantsMiner
+from .LogClustering import LogClustering
+from .LR import LR
+from .SVM import SVM
+from .DecisionTree import DecisionTree
+from .IsolationForest import IsolationForest
+

data/loglizer/preprocessing.py

@@ -0,0 +1,123 @@
+"""
+The interface for data preprocessing.
+
+Authors:
+    LogPAI Team
+
+"""
+
+
+import pandas as pd
+import os
+import numpy as np
+import re
+from collections import Counter
+from scipy.special import expit
+from itertools import compress
+
+
+
+class FeatureExtractor(object):
+
+    def __init__(self):
+        self.idf_vec = None
+        self.mean_vec = None
+        self.events = None
+        self.term_weighting = None
+        self.normalization = None
+        self.oov = None
+
+    def fit_transform(self, X_seq, term_weighting=None, normalization=None, oov=False, min_count=1):
+        """ Fit and transform the data matrix
+
+        Arguments
+        ---------
+            X_seq: ndarray, log sequences matrix
+            term_weighting: None or `tf-idf`
+            normalization: None or `zero-mean`
+            oov: bool, whether to use OOV event
+            min_count: int, the minimal occurrence of events (default 0), only valid when oov=True.
+
+        Returns
+        -------
+            X_new: The transformed data matrix
+        """
+        print('====== Transformed train data summary ======')
+        self.term_weighting = term_weighting
+        self.normalization = normalization
+        self.oov = oov
+
+        X_counts = []
+        for i in range(X_seq.shape[0]):
+            event_counts = Counter(X_seq[i])
+            X_counts.append(event_counts)
+        X_df = pd.DataFrame(X_counts)
+        X_df = X_df.fillna(0)
+        self.events = X_df.columns
+        X = X_df.values
+        if self.oov:
+            oov_vec = np.zeros(X.shape[0])
+            if min_count > 1:
+                idx = np.sum(X > 0, axis=0) >= min_count
+                oov_vec = np.sum(X[:, ~idx] > 0, axis=1)
+                X = X[:, idx]
+                self.events = np.array(X_df.columns)[idx].tolist()
+            X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)])
+        
+        num_instance, num_event = X.shape
+        if self.term_weighting == 'tf-idf':
+            df_vec = np.sum(X > 0, axis=0)
+            self.idf_vec = np.log(num_instance / (df_vec + 1e-8))
+            idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) 
+            X = idf_matrix
+        if self.normalization == 'zero-mean':
+            mean_vec = X.mean(axis=0)
+            self.mean_vec = mean_vec.reshape(1, num_event)
+            X = X - np.tile(self.mean_vec, (num_instance, 1))
+        elif self.normalization == 'sigmoid':
+            X[X != 0] = expit(X[X != 0])
+        X_new = X
+        
+        print('Train data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) 
+        return X_new
+
+    def transform(self, X_seq):
+        """ Transform the data matrix with trained parameters
+
+        Arguments
+        ---------
+            X: log sequences matrix
+            term_weighting: None or `tf-idf`
+
+        Returns
+        -------
+            X_new: The transformed data matrix
+        """
+        print('====== Transformed test data summary ======')
+        X_counts = []
+        for i in range(X_seq.shape[0]):
+            event_counts = Counter(X_seq[i])
+            X_counts.append(event_counts)
+        X_df = pd.DataFrame(X_counts)
+        X_df = X_df.fillna(0)
+        empty_events = set(self.events) - set(X_df.columns)
+        for event in empty_events:
+            X_df[event] = [0] * len(X_df)
+        X = X_df[self.events].values
+        if self.oov:
+            oov_vec = np.sum(X_df[X_df.columns.difference(self.events)].values > 0, axis=1)
+            X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)])
+        
+        num_instance, num_event = X.shape
+        if self.term_weighting == 'tf-idf':
+            idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) 
+            X = idf_matrix
+        if self.normalization == 'zero-mean':
+            X = X - np.tile(self.mean_vec, (num_instance, 1))
+        elif self.normalization == 'sigmoid':
+            X[X != 0] = expit(X[X != 0])
+        X_new = X
+
+        print('Test data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) 
+
+        return X_new, self.events

data/loglizer/utils.py

@@ -0,0 +1,29 @@
+"""
+The utility functions of loglizer
+
+Authors: 
+    LogPAI Team
+
+"""
+
+from sklearn.metrics import precision_recall_fscore_support
+import numpy as np
+
+
+def metrics(y_pred, y_true):
+    """ Calucate evaluation metrics for precision, recall, and f1.
+
+    Arguments
+    ---------
+        y_pred: ndarry, the predicted result list
+        y_true: ndarray, the ground truth label list
+
+    Returns
+    -------
+        precision: float, precision value
+        recall: float, recall value
+        f1: float, f1 measure value
+    """
+    precision, recall, f1, _ = precision_recall_fscore_support(y_true, y_pred, average='binary')
+    return precision, recall, f1
+

data/requirements.txt

@@ -0,0 +1,4 @@
+scikit-learn
+pandas
+streamlit
+openai

data/temp/HDFS_100k.log_structured.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbb13cc937bb35ad683dfcd71929242d7b1e0c01e2a94f79c637847c30a42190
+size 20892087

data_instances.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba4d353dddd4e7ef0de168def9502f38d17dbe95ca5d966c5a6d68aa05b4955
+size 909112

loglizer/dataloader.py

@@ -0,0 +1,268 @@
+"""
+The interface to load log datasets. The datasets currently supported include
+HDFS and BGL.
+
+Authors:
+    LogPAI Team
+
+"""
+
+import pandas as pd
+import os
+import numpy as np
+import re
+from sklearn.utils import shuffle
+from collections import OrderedDict
+
+def _split_data(x_data, y_data=None, train_ratio=0, split_type='uniform'):
+    if split_type == 'uniform' and y_data is not None:
+        pos_idx = y_data > 0
+        x_pos = x_data[pos_idx]
+        y_pos = y_data[pos_idx]
+        x_neg = x_data[~pos_idx]
+        y_neg = y_data[~pos_idx]
+        train_pos = int(train_ratio * x_pos.shape[0])
+        train_neg = int(train_ratio * x_neg.shape[0])
+        x_train = np.hstack([x_pos[0:train_pos], x_neg[0:train_neg]])
+        y_train = np.hstack([y_pos[0:train_pos], y_neg[0:train_neg]])
+        x_test = np.hstack([x_pos[train_pos:], x_neg[train_neg:]])
+        y_test = np.hstack([y_pos[train_pos:], y_neg[train_neg:]])
+    elif split_type == 'sequential':
+        num_train = int(train_ratio * x_data.shape[0])
+        x_train = x_data[0:num_train]
+        x_test = x_data[num_train:]
+        if y_data is None:
+            y_train = None
+            y_test = None
+        else:
+            y_train = y_data[0:num_train]
+            y_test = y_data[num_train:]
+    # Random shuffle
+    indexes = shuffle(np.arange(x_train.shape[0]))
+    x_train = x_train[indexes]
+    if y_train is not None:
+        y_train = y_train[indexes]
+    return (x_train, y_train), (x_test, y_test)
+
+def load_HDFS(log_file, label_file=None, window='session', train_ratio=0.5, split_type='sequential', save_csv=False, window_size=0):
+    """ Load HDFS structured log into train and test data
+
+    Arguments
+    ---------
+        log_file: str, the file path of structured log.
+        label_file: str, the file path of anomaly labels, None for unlabeled data
+        window: str, the window options including `session` (default).
+        train_ratio: float, the ratio of training data for train/test split.
+        split_type: `uniform` or `sequential`, which determines how to split dataset. `uniform` means
+            to split positive samples and negative samples equally when setting label_file. `sequential`
+            means to split the data sequentially without label_file. That is, the first part is for training,
+            while the second part is for testing.
+
+    Returns
+    -------
+        (x_train, y_train): the training data
+        (x_test, y_test): the testing data
+    """
+
+    print('====== Input data summary ======')
+
+    if log_file.endswith('.npz'):
+        # Split training and validation set in a class-uniform way
+        data = np.load(log_file)
+        x_data = data['x_data']
+        y_data = data['y_data']
+        (x_train, y_train), (x_test, y_test) = _split_data(x_data, y_data, train_ratio, split_type)
+
+    elif log_file.endswith('.csv'):
+        assert window == 'session', "Only window=session is supported for HDFS dataset."
+        print("Loading", log_file)
+        struct_log = pd.read_csv(log_file, engine='c',
+                na_filter=False, memory_map=True)
+        data_dict = OrderedDict()
+        for idx, row in struct_log.iterrows():
+            blkId_list = re.findall(r'(blk_-?\d+)', row['Content'])
+            blkId_set = set(blkId_list)
+            for blk_Id in blkId_set:
+                if not blk_Id in data_dict:
+                    data_dict[blk_Id] = []
+                data_dict[blk_Id].append(row['EventId'])
+        data_df = pd.DataFrame(list(data_dict.items()), columns=['BlockId', 'EventSequence'])
+        
+        if label_file:
+            # Split training and validation set in a class-uniform way
+            label_data = pd.read_csv(label_file, engine='c', na_filter=False, memory_map=True)
+            label_data = label_data.set_index('BlockId')
+            label_dict = label_data['Label'].to_dict()
+            data_df['Label'] = data_df['BlockId'].apply(lambda x: 1 if label_dict[x] == 'Anomaly' else 0)
+
+            # Split train and test data
+            (x_train, y_train), (x_test, y_test) = _split_data(data_df['EventSequence'].values, 
+                data_df['Label'].values, train_ratio, split_type)
+        
+            print(y_train.sum(), y_test.sum())
+
+        if save_csv:
+            data_df.to_csv('data_instances.csv', index=False)
+
+        if window_size > 0:
+            x_train, window_y_train, y_train = slice_hdfs(x_train, y_train, window_size)
+            x_test, window_y_test, y_test = slice_hdfs(x_test, y_test, window_size)
+            log = "{} {} windows ({}/{} anomaly), {}/{} normal"
+            print(log.format("Train:", x_train.shape[0], y_train.sum(), y_train.shape[0], (1-y_train).sum(), y_train.shape[0]))
+            print(log.format("Test:", x_test.shape[0], y_test.sum(), y_test.shape[0], (1-y_test).sum(), y_test.shape[0]))
+            return (x_train, window_y_train, y_train), (x_test, window_y_test, y_test)
+
+        if label_file is None:
+            if split_type == 'uniform':
+                split_type = 'sequential'
+                print('Warning: Only split_type=sequential is supported \
+                if label_file=None.'.format(split_type))
+            # Split training and validation set sequentially
+            x_data = data_df['EventSequence'].values
+            (x_train, _), (x_test, _) = _split_data(x_data, train_ratio=train_ratio, split_type=split_type)
+            print('Total: {} instances, train: {} instances, test: {} instances'.format(
+                  x_data.shape[0], x_train.shape[0], x_test.shape[0]))
+            return (x_train, None), (x_test, None), data_df
+    else:
+        raise NotImplementedError('load_HDFS() only support csv and npz files!')
+
+    num_train = x_train.shape[0]
+    num_test = x_test.shape[0]
+    num_total = num_train + num_test
+    num_train_pos = sum(y_train)
+    num_test_pos = sum(y_test)
+    num_pos = num_train_pos + num_test_pos
+
+    print('Total: {} instances, {} anomaly, {} normal' \
+          .format(num_total, num_pos, num_total - num_pos))
+    print('Train: {} instances, {} anomaly, {} normal' \
+          .format(num_train, num_train_pos, num_train - num_train_pos))
+    print('Test: {} instances, {} anomaly, {} normal\n' \
+          .format(num_test, num_test_pos, num_test - num_test_pos))
+
+    return (x_train, y_train), (x_test, y_test)
+
+def slice_hdfs(x, y, window_size):
+    results_data = []
+    print("Slicing {} sessions, with window {}".format(x.shape[0], window_size))
+    for idx, sequence in enumerate(x):
+        seqlen = len(sequence)
+        i = 0
+        while (i + window_size) < seqlen:
+            slice = sequence[i: i + window_size]
+            results_data.append([idx, slice, sequence[i + window_size], y[idx]])
+            i += 1
+        else:
+            slice = sequence[i: i + window_size]
+            slice += ["#Pad"] * (window_size - len(slice))
+            results_data.append([idx, slice, "#Pad", y[idx]])
+    results_df = pd.DataFrame(results_data, columns=["SessionId", "EventSequence", "Label", "SessionLabel"])
+    print("Slicing done, {} windows generated".format(results_df.shape[0]))
+    return results_df[["SessionId", "EventSequence"]], results_df["Label"], results_df["SessionLabel"]
+
+
+
+def load_BGL(log_file, label_file=None, window='sliding', time_interval=60, stepping_size=60, 
+             train_ratio=0.8):
+    """  TODO
+
+    """
+
+
+def bgl_preprocess_data(para, raw_data, event_mapping_data):
+    """ split logs into sliding windows, built an event count matrix and get the corresponding label
+
+    Args:
+    --------
+    para: the parameters dictionary
+    raw_data: list of (label, time)
+    event_mapping_data: a list of event index, where each row index indicates a corresponding log
+
+    Returns:
+    --------
+    event_count_matrix: event count matrix, where each row is an instance (log sequence vector)
+    labels: a list of labels, 1 represents anomaly
+    """
+
+    # create the directory for saving the sliding windows (start_index, end_index), which can be directly loaded in future running
+    if not os.path.exists(para['save_path']):
+        os.mkdir(para['save_path'])
+    log_size = raw_data.shape[0]
+    sliding_file_path = para['save_path']+'sliding_'+str(para['window_size'])+'h_'+str(para['step_size'])+'h.csv'
+
+    #=============divide into sliding windows=========#
+    start_end_index_list = [] # list of tuples, tuple contains two number, which represent the start and end of sliding time window
+    label_data, time_data = raw_data[:,0], raw_data[:, 1]
+    if not os.path.exists(sliding_file_path):
+        # split into sliding window
+        start_time = time_data[0]
+        start_index = 0
+        end_index = 0
+
+        # get the first start, end index, end time
+        for cur_time in time_data:
+            if  cur_time < start_time + para['window_size']*3600:
+                end_index += 1
+                end_time = cur_time
+            else:
+                start_end_pair=tuple((start_index,end_index))
+                start_end_index_list.append(start_end_pair)
+                break
+        # move the start and end index until next sliding window
+        while end_index < log_size:
+            start_time = start_time + para['step_size']*3600
+            end_time = end_time + para['step_size']*3600
+            for i in range(start_index,end_index):
+                if time_data[i] < start_time:
+                    i+=1
+                else:
+                    break
+            for j in range(end_index, log_size):
+                if time_data[j] < end_time:
+                    j+=1
+                else:
+                    break
+            start_index = i
+            end_index = j
+            start_end_pair = tuple((start_index, end_index))
+            start_end_index_list.append(start_end_pair)
+        inst_number = len(start_end_index_list)
+        print('there are %d instances (sliding windows) in this dataset\n'%inst_number)
+        np.savetxt(sliding_file_path,start_end_index_list,delimiter=',',fmt='%d')
+    else:
+        print('Loading start_end_index_list from file')
+        start_end_index_list = pd.read_csv(sliding_file_path, header=None).values
+        inst_number = len(start_end_index_list)
+        print('there are %d instances (sliding windows) in this dataset' % inst_number)
+
+    # get all the log indexes in each time window by ranging from start_index to end_index
+    expanded_indexes_list=[]
+    for t in range(inst_number):
+        index_list = []
+        expanded_indexes_list.append(index_list)
+    for i in range(inst_number):
+        start_index = start_end_index_list[i][0]
+        end_index = start_end_index_list[i][1]
+        for l in range(start_index, end_index):
+            expanded_indexes_list[i].append(l)
+
+    event_mapping_data = [row[0] for row in event_mapping_data]
+    event_num = len(list(set(event_mapping_data)))
+    print('There are %d log events'%event_num)
+
+    #=============get labels and event count of each sliding window =========#
+    labels = []
+    event_count_matrix = np.zeros((inst_number,event_num))
+    for j in range(inst_number):
+        label = 0   #0 represent success, 1 represent failure
+        for k in expanded_indexes_list[j]:
+            event_index = event_mapping_data[k]
+            event_count_matrix[j, event_index] += 1
+            if label_data[k]:
+                label = 1
+                continue
+        labels.append(label)
+    assert inst_number == len(labels)
+    print("Among all instances, %d are anomalies"%sum(labels))
+    assert event_count_matrix.shape[0] == len(labels)
+    return event_count_matrix, labels

loglizer/models/DecisionTree.py

@@ -0,0 +1,78 @@
+"""
+The implementation of the decision tree model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Mike Chen, Alice X. Zheng, Jim Lloyd, Michael I. Jordan, Eric Brewer. 
+        Failure Diagnosis Using Decision Trees. IEEE International Conference 
+        on Autonomic Computing (ICAC), 2004.
+
+"""
+
+import numpy as np
+from sklearn import tree
+from ..utils import metrics
+
+class DecisionTree(object):
+
+    def __init__(self, criterion='gini', max_depth=None, max_features=None, class_weight=None):
+        """ The Invariants Mining model for anomaly detection
+        Arguments
+        ---------
+        See DecisionTreeClassifier API: https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html
+
+        Attributes
+        ----------
+            classifier: object, the classifier for anomaly detection
+
+        """
+        self.classifier = tree.DecisionTreeClassifier(criterion=criterion, max_depth=max_depth,
+                          max_features=max_features, class_weight=class_weight)
+
+    def fit(self, X, y):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        self.classifier.fit(X, y)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = self.classifier.predict(X)
+        return y_pred
+
+    def predict_proba(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = self.classifier.predict_proba(X)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1

loglizer/models/InvariantsMiner.py

@@ -0,0 +1,296 @@
+"""
+The implementation of Invariants Mining model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Jian-Guang Lou, Qiang Fu, Shengqi Yang, Ye Xu, Jiang Li. Mining Invariants 
+        from Console Logs for System Problem Detection. USENIX Annual Technical 
+        Conference (ATC), 2010.
+
+"""
+
+import numpy as np
+from itertools import combinations
+from ..utils import metrics
+
+class InvariantsMiner(object):
+
+    def __init__(self, percentage=0.98, epsilon=0.5, longest_invarant=None, scale_list=[1,2,3]):
+        """ The Invariants Mining model for anomaly detection
+
+        Attributes
+        ----------
+            percentage: float, percentage of samples satisfying the condition that |X_j * V_i| < epsilon
+            epsilon: float, the threshold for estimating the invariant space
+            longest_invarant: int, the specified maximal length of invariant, default to None. Stop 
+                searching when the invariant length is larger than longest_invarant.
+            scale_list: list, the list used to scale the theta of float into integer
+            invariants_dict: dict, dictionary of invariants where key is the selected columns 
+                and value is the weights the of invariant
+        """
+        self.percentage = percentage
+        self.epsilon = epsilon
+        self.longest_invarant = longest_invarant
+        self.scale_list = scale_list
+        self.invariants_dict = None
+
+    def fit(self, X):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        invar_dim = self._estimate_invarant_space(X)
+        self._invariants_search(X, invar_dim)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_sum = np.zeros(X.shape[0])
+        for cols, theta in self.invariants_dict.items():
+            y_sum += np.fabs(np.dot(X[:, cols], np.array(theta)))
+        y_pred = (y_sum > 1e-6).astype(int)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1
+
+    def _estimate_invarant_space(self, X):
+        """ Estimate the dimension of invariant space using SVD decomposition
+
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+            percentage: float, percentage of samples satisfying the condition that |X_j * V_i| < epsilon
+            epsilon: float, the threshold for estimating the invariant space
+
+        Returns
+        -------
+            r: the dimension of invariant space
+        """
+        covariance_matrix = np.dot(X.T, X)
+        U, sigma, V = np.linalg.svd(covariance_matrix)  # SVD decomposition
+        # Start from the right most column of matrix V, sigular values are in ascending order
+        num_instances, num_events = X.shape
+        r = 0
+        for i in range(num_events - 1, -1, -1):
+            zero_count = sum(abs(np.dot(X, U[:, i])) < self.epsilon)
+            if zero_count / float(num_instances) < self.percentage:
+                break
+            r += 1
+        print('Invariant space dimension: {}'.format(r))
+
+        return r
+
+    def _invariants_search(self, X, r):
+        """ Mine invariant relationships from X
+
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+            r: the dimension of invariant space
+        """
+
+        num_instances, num_events = X.shape
+        invariants_dict = dict()  # save the mined Invariants(value) and its corresponding columns(key)
+        search_space = []  # only invariant candidates in this list are valid
+
+        # invariant of only one column (all zero columns)
+        init_cols = sorted([[item] for item in range(num_events)])
+        for col in init_cols:
+            search_space.append(col)
+        init_col_list = init_cols[:]
+        for col in init_cols:
+            if np.count_nonzero(X[:, col]) == 0:
+                invariants_dict[tuple(col)] = [1]
+                search_space.remove(col)
+                init_col_list.remove(col)
+
+        item_list = init_col_list
+        length = 2
+        FLAG_break_loop = False
+        # check invariant of more columns
+        while len(item_list) != 0:
+            if self.longest_invarant and len(item_list[0]) >= self.longest_invarant:
+                break
+            joined_item_list = self._join_set(item_list, length) # generate new invariant candidates
+            for items in joined_item_list:
+                if self._check_candi_valid(items, length, search_space):
+                    search_space.append(items)
+            item_list = []
+            for item in joined_item_list:
+                if tuple(item) in invariants_dict:
+                    continue
+                if item not in search_space:
+                    continue
+                if not self._check_candi_valid(tuple(item), length, search_space) and length > 2:
+                    search_space.remove(item)
+                    continue # an item must be superset of all other subitems in searchSpace, else skip
+                validity, scaled_theta = self._check_invar_validity(X, item)
+                if validity:
+                    self._prune(invariants_dict.keys(), set(item), search_space)
+                    invariants_dict[tuple(item)] = scaled_theta.tolist()
+                    search_space.remove(item)
+                else:
+                    item_list.append(item)
+                if len(invariants_dict) >= r:
+                    FLAG_break_loop = True
+                    break
+            if FLAG_break_loop:
+                break
+            length += 1
+        print('Mined {} invariants: {}\n'.format(len(invariants_dict), invariants_dict))
+        self.invariants_dict = invariants_dict
+
+    def _compute_eigenvector(self, X):
+        """ calculate the smallest eigenvalue and corresponding eigenvector (theta in the paper) 
+            for a given sub_matrix
+
+        Arguments
+        ---------
+            X: the event count matrix (each row is a log sequence vector, each column represents an event)
+
+        Returns
+        -------
+            min_vec: the eigenvector of corresponding minimum eigen value
+            FLAG_contain_zero: whether the min_vec contains zero (very small value)
+        """
+
+        FLAG_contain_zero = False
+        count_zero = 0
+        dot_result = np.dot(X.T, X)
+        U, S, V = np.linalg.svd(dot_result)
+        min_vec = U[:, -1]
+        count_zero = sum(np.fabs(min_vec) < 1e-6)
+        if count_zero != 0:
+            FLAG_contain_zero = True
+        return min_vec, FLAG_contain_zero
+
+
+    def _check_invar_validity(self, X, selected_columns):
+        """ scale the eigenvector of float number into integer, and check whether the scaled number is valid
+
+        Arguments
+        ---------
+            X: the event count matrix (each row is a log sequence vector, each column represents an event)
+            selected_columns: select columns from all column list
+
+        Returns
+        -------
+            validity: whether the selected columns is valid
+            scaled_theta: the scaled theta vector
+        """
+
+        sub_matrix = X[:, selected_columns]
+        inst_num = X.shape[0]
+        validity = False
+        min_theta, FLAG_contain_zero = self._compute_eigenvector(sub_matrix)
+        abs_min_theta = [np.fabs(it) for it in min_theta]
+        if FLAG_contain_zero:
+            return validity, []
+        else:
+            for i in self.scale_list:
+                min_index = np.argmin(abs_min_theta)
+                scale = float(i) / min_theta[min_index]
+                scaled_theta = np.array([round(item * scale) for item in min_theta])
+                scaled_theta[min_index] = i
+                scaled_theta = scaled_theta.T
+                if 0 in np.fabs(scaled_theta):
+                    continue
+                dot_submat_theta = np.dot(sub_matrix, scaled_theta)
+                count_zero = 0
+                for j in dot_submat_theta:
+                    if np.fabs(j) < 1e-8:
+                        count_zero += 1
+                if count_zero >= self.percentage * inst_num:
+                    validity = True
+                    # print('A valid invariant is found: ',scaled_theta, selected_columns)
+                    break
+            return validity, scaled_theta
+
+    def _prune(self, valid_cols, new_item_set, search_space):
+        """ prune invalid combination of columns
+
+        Arguments
+        ---------
+            valid_cols: existing valid column list
+            new_item_set: item set to be merged
+            search_space: the search space that stores possible candidates
+
+        """
+
+        if len(valid_cols) == 0:
+            return
+        for se in valid_cols:
+            intersection = set(se) & new_item_set
+            if len(intersection) == 0:
+                continue
+            union = set(se) | new_item_set
+            for item in list(intersection):
+                diff = sorted(list(union - set([item])))
+                if diff in search_space:
+                    search_space.remove(diff)
+
+
+    def _join_set(self, item_list, length):
+        """ Join a set with itself and returns the n-element (length) itemsets
+
+        Arguments
+        ---------
+            item_list: current list of columns
+            length: generate new items of length
+
+        Returns
+        -------
+            return_list: list of items of length-element
+        """
+
+        set_len = len(item_list)
+        return_list = []
+        for i in range(set_len):
+            for j in range(i + 1, set_len):
+                i_set = set(item_list[i])
+                j_set = set(item_list[j])
+                if len(i_set.union(j_set)) == length:
+                    joined = sorted(list(i_set.union(j_set)))
+                    if joined not in return_list:
+                        return_list.append(joined)
+        return_list = sorted(return_list)
+        return return_list
+
+
+    def _check_candi_valid(self, item, length, search_space):
+        """ check whether an item's subitems are in searchspace
+
+        Arguments
+        ---------
+            item: item to be checked
+            length: the length of item
+            search_space: the search space that stores possible candidates
+
+        Returns
+        -------
+            True or False
+        """
+
+        for subItem in combinations(item, length - 1):
+            if sorted(list(subItem)) not in search_space:
+                return False
+        return True
+

loglizer/models/IsolationForest.py

@@ -0,0 +1,96 @@
+"""
+The implementation of IsolationForest model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Fei Tony Liu, Kai Ming Ting, Zhi-Hua Zhou. Isolation Forest. International
+        Conference on Data Mining (ICDM), 2008.
+
+"""
+
+
+
+
+
+import numpy as np
+from sklearn.ensemble import IsolationForest as iForest
+from ..utils import metrics
+
+class IsolationForest(iForest):
+
+    def __init__(self, n_estimators=100, max_samples='auto', contamination=0.03, **kwargs):
+        """ The IsolationForest model for anomaly detection
+
+        Arguments
+        ---------
+            n_estimators : int, optional (default=100). The number of base estimators in the ensemble.
+            max_samples : int or float, optional (default="auto")
+                The number of samples to draw from X to train each base estimator.
+                    - If int, then draw max_samples samples.
+                    - If float, then draw max_samples * X.shape[0] samples.
+                    - If "auto", then max_samples=min(256, n_samples).
+                If max_samples is larger than the number of samples provided, all samples will be used 
+                for all trees (no sampling).
+            contamination : float in (0., 0.5), optional (default='auto')
+                The amount of contamination of the data set, i.e. the proportion of outliers in the data 
+                set. Used when fitting to define the threshold on the decision function. If 'auto', the 
+                decision function threshold is determined as in the original paper.
+            max_features : int or float, optional (default=1.0)
+                The number of features to draw from X to train each base estimator.
+                    - If int, then draw max_features features.
+                    - If float, then draw max_features * X.shape[1] features.
+            bootstrap : boolean, optional (default=False)
+                If True, individual trees are fit on random subsets of the training data sampled with replacement. 
+                If False, sampling without replacement is performed.
+            n_jobs : int or None, optional (default=None)
+                The number of jobs to run in parallel for both fit and predict. None means 1 unless in a 
+                joblib.parallel_backend context. -1 means using all processors. 
+            random_state : int, RandomState instance or None, optional (default=None)
+                If int, random_state is the seed used by the random number generator; 
+                If RandomState instance, random_state is the random number generator; 
+                If None, the random number generator is the RandomState instance used by np.random.
+        
+        Reference
+        ---------
+            For more information, please visit https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html
+        """
+
+        super(IsolationForest, self).__init__(n_estimators=n_estimators, max_samples=max_samples, 
+            contamination=contamination, **kwargs)
+
+
+    def fit(self, X):
+        """
+        Auguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+
+        print('====== Model summary ======')
+        super(IsolationForest, self).fit(X)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = super(IsolationForest, self).predict(X)
+        y_pred = np.where(y_pred > 0, 0, 1)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1
+

loglizer/models/LR.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+The implementation of the logistic regression model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Peter Bodík, Moises Goldszmidt, Armando Fox, Hans Andersen. Fingerprinting 
+        the Datacenter: Automated Classification of Performance Crises. The European 
+        Conference on Computer Systems (EuroSys), 2010.
+
+"""
+
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from ..utils import metrics
+
+class LR(object):
+
+    def __init__(self, penalty='l2', C=100, tol=0.01, class_weight=None, max_iter=100):
+        """ The Invariants Mining model for anomaly detection
+
+        Attributes
+        ----------
+            classifier: object, the classifier for anomaly detection
+        """
+        self.classifier = LogisticRegression(penalty=penalty, C=C, tol=tol, class_weight=class_weight,
+                                             max_iter=max_iter)
+
+    def fit(self, X, y):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        self.classifier.fit(X, y)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        y_pred = self.classifier.predict(X)
+        return y_pred
+
+    def predict_proba(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        y_pred = self.classifier.predict_proba(X)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1

loglizer/models/LogClustering.py

@@ -0,0 +1,137 @@
+"""
+The implementation of Log Clustering model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Qingwei Lin, Hongyu Zhang, Jian-Guang Lou, Yu Zhang, Xuewei Chen. Log Clustering 
+        based Problem Identification for Online Service Systems. International Conference
+        on Software Engineering (ICSE), 2016.
+
+"""
+
+import numpy as np
+import pprint
+from scipy.special import expit
+from numpy import linalg as LA
+from scipy.cluster.hierarchy import linkage, fcluster
+from scipy.spatial.distance import pdist, squareform
+from ..utils import metrics
+
+
+class LogClustering(object):
+
+    def __init__(self, max_dist=0.3, anomaly_threshold=0.3, mode='online', num_bootstrap_samples=1000):
+        """
+        Attributes
+        ----------
+            max_dist: float, the threshold to stop the clustering process
+            anomaly_threshold: float, the threshold for anomaly detection
+            mode: str, 'offline' or 'online' mode for clustering
+            num_bootstrap_samples: int, online clustering starts with a bootstraping process, which
+                determines the initial cluster representatives offline using a subset of samples 
+            representatives: ndarray, the representative samples of clusters, of shape 
+                num_clusters-by-num_events
+            cluster_size_dict: dict, the size of each cluster, used to update representatives online 
+        """
+        self.max_dist = max_dist
+        self.anomaly_threshold = anomaly_threshold
+        self.mode = mode
+        self.num_bootstrap_samples = num_bootstrap_samples
+        self.representatives = list()
+        self.cluster_size_dict = dict()
+
+    def fit(self, X):   
+        print('====== Model summary ======')         
+        if self.mode == 'offline':
+            # The offline mode can process about 10K samples only due to huge memory consumption.
+            self._offline_clustering(X)
+        elif self.mode == 'online':
+            # Bootstrapping phase
+            if self.num_bootstrap_samples > 0:
+                X_bootstrap = X[0:self.num_bootstrap_samples, :]
+                self._offline_clustering(X_bootstrap)
+            # Online learning phase
+            if X.shape[0] > self.num_bootstrap_samples:
+                self._online_clustering(X)
+
+    def predict(self, X):
+        y_pred = np.zeros(X.shape[0])
+        for i in range(X.shape[0]):
+            min_dist, min_index = self._get_min_cluster_dist(X[i, :])
+            if min_dist > self.anomaly_threshold:
+                y_pred[i] = 1
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n' \
+              .format(precision, recall, f1))
+        return precision, recall, f1
+
+    def _offline_clustering(self, X):
+        print('Starting offline clustering...')
+        p_dist = pdist(X, metric=self._distance_metric)
+        Z = linkage(p_dist, 'complete')
+        cluster_index = fcluster(Z, self.max_dist, criterion='distance')
+        self._extract_representatives(X, cluster_index)
+        print('Processed {} instances.'.format(X.shape[0]))
+        print('Found {} clusters offline.\n'.format(len(self.representatives)))
+        # print('The representive vectors are:')
+        # pprint.pprint(self.representatives.tolist())
+
+    def _extract_representatives(self, X, cluster_index):
+        num_clusters = len(set(cluster_index))
+        for clu in range(num_clusters):
+            clu_idx = np.argwhere(cluster_index == clu + 1)[:, 0]
+            self.cluster_size_dict[clu] = clu_idx.shape[0]
+            repre_center = np.average(X[clu_idx, :], axis=0)
+            self.representatives.append(repre_center)
+
+    def _online_clustering(self, X):
+        print("Starting online clustering...")
+        for i in range(self.num_bootstrap_samples, X.shape[0]):
+            if (i + 1) % 2000 == 0:
+                print('Processed {} instances.'.format(i + 1))
+            instance_vec = X[i, :]
+            if len(self.representatives) > 0:
+                min_dist, clu_id = self._get_min_cluster_dist(instance_vec)
+                if min_dist <= self.max_dist:
+                    self.cluster_size_dict[clu_id] += 1
+                    self.representatives[clu_id] = self.representatives[clu_id] \
+                                                 + (instance_vec - self.representatives[clu_id]) \
+                                                 / self.cluster_size_dict[clu_id]
+                    continue
+            self.cluster_size_dict[len(self.representatives)] = 1
+            self.representatives.append(instance_vec)
+        print('Processed {} instances.'.format(X.shape[0]))
+        print('Found {} clusters online.\n'.format(len(self.representatives)))
+        # print('The representive vectors are:')
+        # pprint.pprint(self.representatives.tolist())
+
+    def _distance_metric(self, x1, x2):
+        norm= LA.norm(x1) * LA.norm(x2)
+        distance = 1 - np.dot(x1, x2) / (norm + 1e-8)
+        if distance < 1e-8:
+            distance = 0
+        return distance
+
+    def _get_min_cluster_dist(self, instance_vec):
+        min_index = -1
+        min_dist = float('inf')
+        for i in range(len(self.representatives)):
+            cluster_rep = self.representatives[i]
+            dist = self._distance_metric(instance_vec, cluster_rep)
+            if dist < 1e-8:
+                min_dist = 0
+                min_index = i
+                break
+            elif dist < min_dist:
+                min_dist = dist
+                min_index = i
+        return min_dist, min_index
+
+

loglizer/models/PCA.py

@@ -0,0 +1,105 @@
+"""
+The implementation of PCA model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Wei Xu, Ling Huang, Armando Fox, David Patterson, Michael I. Jordan. 
+        Large-Scale System Problems Detection by Mining Console Logs. ACM 
+        Symposium on Operating Systems Principles (SOSP), 2009.
+
+"""
+
+import numpy as np
+from ..utils import metrics
+
+class PCA(object):
+
+    def __init__(self, n_components=0.95, threshold=None, c_alpha=3.2905):
+        """ The PCA model for anomaly detection
+
+        Attributes
+        ----------
+            proj_C: The projection matrix for projecting feature vector to abnormal space
+            n_components: float/int, number of principal compnents or the variance ratio they cover
+            threshold: float, the anomaly detection threshold. When setting to None, the threshold 
+                is automatically caculated using Q-statistics
+            c_alpha: float, the c_alpha parameter for caculating anomaly detection threshold using 
+                Q-statistics. The following is lookup table for c_alpha:
+                c_alpha = 1.7507; # alpha = 0.08
+                c_alpha = 1.9600; # alpha = 0.05
+                c_alpha = 2.5758; # alpha = 0.01
+                c_alpha = 2.807; # alpha = 0.005
+                c_alpha = 2.9677;  # alpha = 0.003
+                c_alpha = 3.2905;  # alpha = 0.001
+                c_alpha = 3.4808;  # alpha = 0.0005
+                c_alpha = 3.8906;  # alpha = 0.0001
+                c_alpha = 4.4172;  # alpha = 0.00001
+        """
+
+        self.proj_C = None
+        self.components = None
+        self.n_components = n_components
+        self.threshold = threshold
+        self.c_alpha = c_alpha
+
+
+    def fit(self, X):
+        """
+        Auguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+
+        print('====== Model summary ======')
+        num_instances, num_events = X.shape
+        X_cov = np.dot(X.T, X) / float(num_instances)
+        U, sigma, V = np.linalg.svd(X_cov)
+        n_components = self.n_components
+        if n_components < 1:
+            total_variance = np.sum(sigma)
+            variance = 0
+            for i in range(num_events):
+                variance += sigma[i]
+                if variance / total_variance >= n_components:
+                    break
+            n_components = i + 1
+
+        P = U[:, :n_components]
+        I = np.identity(num_events, int)
+        self.components = P
+        self.proj_C = I - np.dot(P, P.T)
+        print('n_components: {}'.format(n_components))
+        print('Project matrix shape: {}-by-{}'.format(self.proj_C.shape[0], self.proj_C.shape[1]))
+
+        if not self.threshold:
+            # Calculate threshold using Q-statistic. Information can be found at:
+            # http://conferences.sigcomm.org/sigcomm/2004/papers/p405-lakhina111.pdf
+            phi = np.zeros(3)
+            for i in range(3):
+                for j in range(n_components, num_events):
+                    phi[i] += np.power(sigma[j], i + 1)
+            h0 = 1.0 - 2 * phi[0] * phi[2] / (3.0 * phi[1] * phi[1])
+            self.threshold = phi[0] * np.power(self.c_alpha * np.sqrt(2 * phi[1] * h0 * h0) / phi[0]
+                                               + 1.0 + phi[1] * h0 * (h0 - 1) / (phi[0] * phi[0]), 
+                                               1.0 / h0)
+        print('SPE threshold: {}\n'.format(self.threshold))
+
+    def predict(self, X):
+        assert self.proj_C is not None, 'PCA model needs to be trained before prediction.'
+        y_pred = np.zeros(X.shape[0])
+        for i in range(X.shape[0]):
+            y_a = np.dot(self.proj_C, X[i, :])
+            SPE = np.dot(y_a, y_a)
+            if SPE > self.threshold:
+                y_pred[i] = 1
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1
+

loglizer/models/SVM.py

@@ -0,0 +1,64 @@
+"""
+The implementation of the SVM model for anomaly detection.
+
+Authors: 
+    LogPAI Team
+
+Reference: 
+    [1] Yinglung Liang, Yanyong Zhang, Hui Xiong, Ramendra Sahoo. Failure Prediction 
+        in IBM BlueGene/L Event Logs. IEEE International Conference on Data Mining
+        (ICDM), 2007.
+
+"""
+
+import numpy as np
+from sklearn import svm
+from ..utils import metrics
+
+class SVM(object):
+
+    def __init__(self, penalty='l1', tol=0.1, C=1, dual=False, class_weight=None, 
+                 max_iter=100):
+        """ The Invariants Mining model for anomaly detection
+        Arguments
+        ---------
+        See SVM API: https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html
+        
+        Attributes
+        ----------
+            classifier: object, the classifier for anomaly detection
+
+        """
+        self.classifier = svm.LinearSVC(penalty=penalty, tol=tol, C=C, dual=dual, 
+                                        class_weight=class_weight, max_iter=max_iter)
+
+    def fit(self, X, y):
+        """
+        Arguments
+        ---------
+            X: ndarray, the event count matrix of shape num_instances-by-num_events
+        """
+        print('====== Model summary ======')
+        self.classifier.fit(X, y)
+
+    def predict(self, X):
+        """ Predict anomalies with mined invariants
+
+        Arguments
+        ---------
+            X: the input event count matrix
+
+        Returns
+        -------
+            y_pred: ndarray, the predicted label vector of shape (num_instances,)
+        """
+        
+        y_pred = self.classifier.predict(X)
+        return y_pred
+
+    def evaluate(self, X, y_true):
+        print('====== Evaluation summary ======')
+        y_pred = self.predict(X)
+        precision, recall, f1 = metrics(y_pred, y_true)
+        print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
+        return precision, recall, f1

loglizer/models/__init__.py

@@ -0,0 +1,8 @@
+from .PCA import PCA
+from .InvariantsMiner import InvariantsMiner
+from .LogClustering import LogClustering
+from .LR import LR
+from .SVM import SVM
+from .DecisionTree import DecisionTree
+from .IsolationForest import IsolationForest
+