Merge branch 'main' of https://huggingface.co/spaces/gstdl/screener-saham-demo

app/dataset/Klasifikasi Industri Perusahaan Tercatat.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16d31e3ed82592828347ad047cea086300dc43b039518bfa1c1fd95069b7f12d
+size 1378039

app/dataset/patterns.json

@@ -0,0 +1,63 @@
+{
+    "CDL2CROWS":"Two Crows",
+     "CDL3BLACKCROWS":"Three Black Crows",
+     "CDL3INSIDE":"Three Inside Up/Down",
+     "CDL3LINESTRIKE":"Three-Line Strike",
+     "CDL3OUTSIDE":"Three Outside Up/Down",
+     "CDL3STARSINSOUTH":"Three Stars In The South",
+     "CDL3WHITESOLDIERS":"Three Advancing White Soldiers",
+     "CDLABANDONEDBABY":"Abandoned Baby",
+     "CDLADVANCEBLOCK":"Advance Block",
+     "CDLBELTHOLD":"Belt-hold",
+     "CDLBREAKAWAY":"Breakaway",
+     "CDLCLOSINGMARUBOZU":"Closing Marubozu",
+     "CDLCONCEALBABYSWALL":"Concealing Baby Swallow",
+     "CDLCOUNTERATTACK":"Counterattack",
+     "CDLDARKCLOUDCOVER":"Dark Cloud Cover",
+     "CDLDOJI":"Doji",
+     "CDLDOJISTAR":"Doji Star",
+     "CDLDRAGONFLYDOJI":"Dragonfly Doji",
+     "CDLENGULFING":"Engulfing Pattern",
+     "CDLEVENINGDOJISTAR":"Evening Doji Star",
+     "CDLEVENINGSTAR":"Evening Star",
+     "CDLGAPSIDESIDEWHITE":"Up/Down-gap side-by-side white lines",
+     "CDLGRAVESTONEDOJI":"Gravestone Doji",
+     "CDLHAMMER":"Hammer",
+     "CDLHANGINGMAN":"Hanging Man",
+     "CDLHARAMI":"Harami Pattern",
+     "CDLHARAMICROSS":"Harami Cross Pattern",
+     "CDLHIGHWAVE":"High-Wave Candle",
+     "CDLHIKKAKE":"Hikkake Pattern",
+     "CDLHIKKAKEMOD":"Modified Hikkake Pattern",
+     "CDLHOMINGPIGEON":"Homing Pigeon",
+     "CDLIDENTICAL3CROWS":"Identical Three Crows",
+     "CDLINNECK":"In-Neck Pattern",
+     "CDLINVERTEDHAMMER":"Inverted Hammer",
+     "CDLKICKING":"Kicking",
+     "CDLKICKINGBYLENGTH":"Kicking - bull/bear determined by the longer marubozu",
+     "CDLLADDERBOTTOM":"Ladder Bottom",
+     "CDLLONGLEGGEDDOJI":"Long Legged Doji",
+     "CDLLONGLINE":"Long Line Candle",
+     "CDLMARUBOZU":"Marubozu",
+     "CDLMATCHINGLOW":"Matching Low",
+     "CDLMATHOLD":"Mat Hold",
+     "CDLMORNINGDOJISTAR":"Morning Doji Star",
+     "CDLMORNINGSTAR":"Morning Star",
+     "CDLONNECK":"On-Neck Pattern",
+     "CDLPIERCING":"Piercing Pattern",
+     "CDLRICKSHAWMAN":"Rickshaw Man",
+     "CDLRISEFALL3METHODS":"Rising/Falling Three Methods",
+     "CDLSEPARATINGLINES":"Separating Lines",
+     "CDLSHOOTINGSTAR":"Shooting Star",
+     "CDLSHORTLINE":"Short Line Candle",
+     "CDLSPINNINGTOP":"Spinning Top",
+     "CDLSTALLEDPATTERN":"Stalled Pattern",
+     "CDLSTICKSANDWICH":"Stick Sandwich",
+     "CDLTAKURI":"Takuri (Dragonfly Doji with very long lower shadow)",
+     "CDLTASUKIGAP":"Tasuki Gap",
+     "CDLTHRUSTING":"Thrusting Pattern",
+     "CDLTRISTAR":"Tristar Pattern",
+     "CDLUNIQUE3RIVER":"Unique 3 River",
+     "CDLUPSIDEGAP2CROWS":"Upside Gap Two Crows",
+     "CDLXSIDEGAP3METHODS":"Upside/Downside Gap Three Methods"
+ }

app/dataset/pull_data.py

@@ -0,0 +1,140 @@
+import tabula
+import yfinance as yfi
+import sqlite3
+import pandas as pd
+import json
+import talib
+import time
+import datetime
+import warnings
+
+warnings.filterwarnings("ignore")
+
+with open("patterns.json", "r") as f:
+    patterns = json.load(f)
+
+# update_time = datetime.datetime.now()
+# dummy update time
+update_time = "2021-05-01 15:20:03.672744"
+
+def find_patterns(df):
+    result = pd.DataFrame(
+        columns=[
+            "Date",
+            "Kode",
+            "Pattern",
+            "Pattern_Score",
+            "Open_Close_Change",
+            "High_Low_Change",
+        ]
+    )
+    for attr, pattern in patterns.items():
+        scores = getattr(talib, attr)(df["Open"], df["High"], df["Low"], df["Close"])
+        mask = scores != 0
+        temp_result = df[mask]
+        if len(temp_result) > 0:
+            temp_result = temp_result.assign(
+                Open_Close_Change=(temp_result["Close"] - temp_result["Open"]) / temp_result["Open"],
+                High_Low_Change=(temp_result["High"] - temp_result["Low"]) / temp_result["Low"],
+                Pattern=[pattern] * len(temp_result),
+                Pattern_Score=scores[mask].values,
+            )[result.columns]
+            result = result.append(temp_result)
+    result = result.assign(time_updated = update_time)
+    return result
+
+
+def pull_data_yfi():
+    start = time.time()
+    with sqlite3.connect("ihsg.db") as con:
+        tickers = pd.read_sql(
+        """
+            SELECT Kode FROM list_perusahaan
+            WHERE Kode != "IHSG"
+        """,
+        con=con,
+        ).values.flatten()
+        ihsg = (
+            yfi.download("^JKSE", start="2015-01-01", end="2021-05-01", progress=False)
+            .reset_index()
+            .dropna()
+            .assign(Kode="IHSG")
+        )
+        ihsg = ihsg[["Date", "Kode", "Open", "High", "Low", "Close", "Volume"]]
+        ihsg = ihsg.assign(time_updated = update_time)
+        ihsg.to_sql("historical", if_exists="replace", con=con, index=False)
+        pattern_search = find_patterns(ihsg)
+        pattern_search.to_sql("patterns", if_exists="replace", con=con, index=False)
+        for i in range(0, len(tickers), 50):
+            ticker = [f"{kode}.JK" for kode in tickers[i : i + 50]]
+            df = (
+                yfi.download(ticker, start="2015-01-01", end="2021-05-01", progress=False)
+                .T.unstack(level=1)
+                .T.reset_index()
+                .dropna()
+                .rename(columns={"level_1": "Kode"})
+            )
+            df = df[["Date", "Kode", "Open", "High", "Low", "Close", "Volume"]]
+            df["Kode"] = df["Kode"].str.replace(".JK", "")
+            for j, kode in enumerate(df["Kode"].unique()):
+                print(f"Finding Patterns for {kode} #{i+j+1}\t\t time elapsed = {time.time() - start:.2f} s")
+                pattern_search = find_patterns(df[df["Kode"] == kode])
+                pattern_search.to_sql("patterns", if_exists="append", con=con, index=False)
+            df = df.assign(time_updated = update_time)
+            df.to_sql("historical", if_exists="append", con=con, index=False)
+
+def pull_data_klasifikasi_industri():
+    with sqlite3.connect("ihsg.db") as con:
+        cur = con.cursor()
+        cur.execute("DROP TABLE IF EXISTS list_perusahaan")
+        cur.execute("""
+            CREATE TABLE list_perusahaan (
+                Kode VARCHAR(4), 
+                Nama TEXT, 
+                Sektor TEXT,
+                Instrumen TEXT)
+        """)
+        cur.execute("""
+            INSERT INTO list_perusahaan VALUES
+            ('IHSG', 'Indeks Harga Saham Gabungan', NULL, 'Indeks')
+        """)
+        # TODO: Change Schema from Star Schema to Snowflake Schema
+        # list_perusahaan table will be the dimension table for sector and sub-sector fact tables
+        # note: list_perusahaan table is a dimension table for historical fact table
+        
+        dfs = tabula.read_pdf("Klasifikasi Industri Perusahaan Tercatat.pdf", pages="all", stream=True)
+        # print(len(dfs))
+        for df in dfs:
+            kode, nama, sektor = None, None, None
+            for row in df.iloc[2:,:].itertuples():
+                if kode is not None and pd.notna(row[2]):
+                    cur.execute(f"""
+                        INSERT INTO list_perusahaan VALUES
+                        ('{kode}', '{nama}', '{sektor}', 'Saham')
+                    """)
+                    kode, nama, sektor = None, None, None
+                elif kode is not None and pd.isna(row[2]):
+                    if pd.notna(row[3]):
+                        nama += " " + row[3] 
+                    if pd.notna(row[5]):
+                        sektor += " " + row[5]
+                if kode is None and nama is None and sektor is None and pd.notna(row[2]):
+                    if "saham" in row[8].lower():
+                        kode = row[2]
+                        nama = row[3]
+                        sektor = row[5]
+            else:
+                if kode is not None:
+                    cur.execute(f"""
+                        INSERT INTO list_perusahaan VALUES
+                        ('{kode}', '{nama}', '{sektor}', 'Saham')
+                    """)
+            print("INSERTION RESULT: \n")
+            print(pd.read_sql("SELECT * FROM list_perusahaan", con=con).tail(2))
+            print(pd.read_sql("SELECT * FROM list_perusahaan", con=con).shape)
+            print("\n\n*--\n")
+        con.commit()
+
+if __name__ == "__main__":
+    pull_data_klasifikasi_industri()
+    pull_data_yfi()

app/helper_script.py

@@ -0,0 +1,176 @@
+from bokeh.plotting import figure
+from bokeh.models import ColumnDataSource, HoverTool, Arrow, NormalHead
+from bokeh.palettes import Spectral4
+from bokeh.embed import components
+import sqlite3
+import pandas as pd
+
+
+def get_tickers(pattern, last_dates=1):
+    # connect to database
+    with sqlite3.connect("dataset/ihsg.db") as con:
+
+        # retrieve data from database
+        tickers = pd.read_sql(f"""
+            SELECT Kode 
+            FROM patterns
+            WHERE Date IN (
+                SELECT Date 
+                FROM (
+                    SELECT Date, ROW_NUMBER() OVER(ORDER BY Date DESC) AS rnk
+                    FROM historical
+                    WHERE Kode = 'IHSG'
+                ) a 
+                WHERE rnk <= {last_dates + 1}
+            )
+            AND Pattern = '{pattern}'
+            ORDER BY Pattern_Score DESC, Open_Close_Change DESC, High_Low_Change DESC
+        """,
+        con=con,
+        ).iloc[:, 0].to_list()
+
+    return tickers
+
+def get_data(kode, pattern):
+
+    # connect to database
+    with sqlite3.connect("dataset/ihsg.db") as con:
+
+        # retrieve data from database
+        df = pd.read_sql(f"""
+            SELECT *
+            FROM historical
+            WHERE Kode = '{kode}'
+            ORDER BY Date
+        """, 
+        con=con, 
+        parse_dates=['Date'],
+        )
+        
+        # df = pd.read_sql(f"""
+        #     SELECT 
+        #         historical.Date,
+        #         historical.Open,
+        #         historical.High,
+        #         historical.Low,
+        #         historical.Close, 
+        #         patterns.Pattern_Score
+        #     FROM historical
+        #     LEFT JOIN (
+        #         SELECT Date, Kode, Pattern_Score
+        #         FROM patterns
+        #         WHERE Pattern = '{pattern}'
+        #     ) AS patterns
+        #     USING(Kode, Date)
+        #     WHERE Kode = '{kode}'
+        #     ORDER BY Date
+        # """, 
+        # con=con, 
+        # parse_dates=['Date'],
+        # )
+
+        nama = pd.read_sql(
+            f"SELECT Nama FROM list_perusahaan WHERE Kode = '{kode}'", 
+            con=con,
+        ).values[0][0]
+
+    return df, nama
+
+def plot_candlestick(df, nama, kode):
+    
+    # calculate simple moving average
+    for period in [5,20,200]:
+        df[f'sma{period}'] = df['Close'].rolling(period, period).mean()
+        
+    # Prepare data for plotting
+    cds = ColumnDataSource(df)
+    cds_inc = ColumnDataSource(df[df["Close"] >= df["Open"]])
+    cds_dec = ColumnDataSource(df[df["Open"] > df["Close"]])
+    
+    # assign figure canvas to variable p
+    x_range = (max(len(df) - 60.5, 0), len(df))
+    p = figure(
+        tools="pan,zoom_in,zoom_out,box_zoom,undo,redo,reset,save", 
+        plot_width=600, 
+        plot_height=400,
+        title = f"{kode}\t({nama})",
+        x_range= x_range,
+        y_range= (
+            df.loc[x_range[0]//1-5:x_range[1], ["Open", "High", "Low", "Close", "sma5", "sma20", "sma200"]].min().min() * 0.875, 
+            df.loc[x_range[0]//1-5:x_range[1], ["Open", "High", "Low", "Close", "sma5", "sma20", "sma200"]].max().max() * 1.125
+        )
+    )
+    
+    # xaxis setup
+    p.xaxis.major_label_overrides = {
+    i: date.strftime('%d %b %Y') for i, date in enumerate(df["Date"])
+    }
+    p.xaxis.bounds = (0, df.index[-1])
+    p.xaxis.major_label_orientation = (22/7)/4
+    p.grid.grid_line_alpha=0.3
+    
+    # # plot pattern arrow
+    # for idx in df[df["Pattern_Score"].notna()].tail().index:
+    #     row = df.loc[idx, ["Open", "High", "Low", "Close"]]
+    #     x_start = row.min()
+    #     if x_start < 200:
+    #         x_start -= 2
+    #         x_end = x_start - 4
+    #     elif x_start < 500:
+    #         x_start -= 4
+    #         x_end = x_start - 4
+    #     else:
+    #         x_start -= 8
+    #         x_end = x_start - 6
+    #     p.add_layout(Arrow(
+    #         end=NormalHead(fill_color="black"), 
+    #         line_color="black",
+    #         x_start = x_start,
+    #         x_end = x_end,
+    #         y_start = idx,
+    #         y_end=idx
+    #     ))
+        
+
+    # plot candlestick wicks with HoverTool
+    p.add_tools(HoverTool(
+        renderers=[p.segment("index", "High", "index", "Low", source=cds, color="black", line_width=1)],
+        tooltips=[
+            ("Date","@Date{%F}"),
+            ("Open","@Open{0.2f}"),
+            ("High", "@High{0.2f}"),
+            ("Low", "@Low{0.2f}"),
+            ("Close", "@Close{0.2f}"),
+        ],
+        formatters={"@Date":"datetime"}
+    ))
+
+    # plot candlestick bars
+    for data, color in [(cds_inc, "#26a69a"), (cds_dec, "#ef5350")]:
+        p.vbar("index", 0.5, "Open", "Close", source=data, fill_color=color, line_color="black", line_width=1)
+    
+    # plot moving average with HoverTool
+    for period, color in zip([5,20,200], Spectral4):
+        p.add_tools(HoverTool(
+            renderers=[p.line(
+                "index", 
+                f"sma{period}", 
+                source=cds, 
+                line_width=2, 
+                alpha=0.8, 
+                color=color, 
+                legend_label=f'SMA {period}\t')],
+            tooltips=[
+                (f"SMA {period}", "@sma%s{0.2f}" %(period)),
+            ],
+        ))
+    
+    # legend setup
+    p.legend.location = "top_left"
+    p.legend.click_policy="hide"
+    p.legend.orientation="horizontal"
+    
+    # generate script and div
+    script, div = components(p)
+
+    return script, div

app/requirements.txt

@@ -0,0 +1,6 @@
+Flask==1.1.1
+gunicorn==20.1.0
+pandas==1.2.5
+yfinance==0.1.59
+TA-Lib==0.4.20
+bokeh==2.3.2

app/templates/index.html

@@ -0,0 +1,52 @@
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="./assets/css/styles.css" type="text/css">
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
+    <script type="text/javascript">
+        window.onload = function() {
+        $("#loading").hide();
+        };
+        function loading(){
+            $("#content").hide();
+            $("#loading").show();
+        }
+    </script>
+    {% block head %}{% endblock %}
+    <title>Candlestick Screener</title>
+</head>
+<body>
+    <div id="loading">
+        <!-- <img src="/assets/img/loading.gif"> -->
+        <img src="https://cdn.dribbble.com/users/115601/screenshots/5356365/loading.gif">
+    </div>
+    <div id="content">
+    <h1>Candlestick Screener</h1>
+    <form>
+        <label for="pattern">Find pattern:</label>
+        <select name="pattern">
+            {% if not selected %}
+                <option value="" selected disabled hidden></option>
+            {% endif %}
+            {% for pattern in patterns %}
+                {% if selected == pattern %}
+                    <option value={{ pattern }} selected>{{ patterns[pattern] }}</option>
+                {% else %}
+                    <option value={{ pattern }}>{{ patterns[pattern] }}</option>
+                {% endif %}
+            {% endfor %}
+        </select>
+        <input type="submit" value="search" onclick="loading();"/>
+    </form>
+    <form method="POST">
+        Database updated at: {{ last_update_time }}
+        <button type="submit" value="Update Database" onclick="loading();">
+            Update Database
+        </button>
+    </form>
+    {% block contents %}{% endblock %}
+    </div>
+</body>
+</html>

app/templates/no_pattern_found.html

@@ -0,0 +1,7 @@
+{% extends "index.html" %}
+
+{% block contents %}
+
+    <h3>Candlestick pattern is not present in the last candlesticks for all stocks</h3>
+
+{% endblock %}

app/templates/plot.html

@@ -0,0 +1,15 @@
+{% extends "index.html" %}
+
+{% block head %}
+    {{ js_resources|indent(4)|safe }}
+    {{ css_resources|indent(4)|safe }}
+    {% for plot_script in plot_scripts %}
+        {{ plot_script|indent(4)|safe }}
+    {% endfor %}
+{% endblock %}
+
+{% block contents %}
+    {% for plot_div in plot_divs %}
+        {{ plot_div | safe }}
+    {% endfor %}
+{% endblock %}

app/update_database.py

@@ -0,0 +1,116 @@
+import pandas as pd
+import sqlite3
+import json
+import datetime
+import time
+import yfinance as yfi
+import talib
+import warnings
+warnings.filterwarnings("ignore")
+
+# retrieve pattern data
+with open("dataset/patterns.json") as f:
+    patterns = json.load(f)
+
+
+def get_last_update_time():
+    with sqlite3.connect("dataset/ihsg.db") as con:
+        return pd.read_sql("SELECT MAX(time_updated) FROM historical", con=con).values[0][0][:19]
+
+def find_patterns(df):
+    result = pd.DataFrame(
+        columns=[
+            "Date",
+            "Kode",
+            "Pattern",
+            "Pattern_Score",
+            "Open_Close_Change",
+            "High_Low_Change",
+        ]
+    )
+    for attr, pattern in patterns.items():
+        scores = getattr(talib, attr)(df["Open"], df["High"], df["Low"], df["Close"])
+        mask = scores != 0
+        temp_result = df[mask]
+        if len(temp_result) > 0:
+            temp_result = temp_result.assign(
+                Open_Close_Change=(temp_result["Close"] - temp_result["Open"]) / temp_result["Open"],
+                High_Low_Change=(temp_result["High"] - temp_result["Low"]) / temp_result["Low"],
+                Pattern=[pattern] * len(temp_result),
+                Pattern_Score=scores[mask].values,
+            )[result.columns]
+            result = result.append(temp_result)
+    result = result.assign(time_updated = datetime.datetime.now())
+    return result
+
+
+def update_database():
+
+    with sqlite3.connect("dataset/ihsg.db") as con:
+        start_date = datetime.datetime.strptime(
+            pd.read_sql("SELECT MAX(Date) FROM historical", con=con).values[0][0], 
+            "%Y-%m-%d %H:%M:%S"
+        )
+        start_date += pd.offsets.DateOffset(days=1)
+        start_date = datetime.datetime.strftime(start_date, "%Y-%m-%d")
+        end_date = datetime.datetime.now()
+        if (end_date.hour) < 15:
+            end_date -= pd.offsets.DateOffset(days = 1)
+        end_date = datetime.datetime.strftime(end_date, "%Y-%m-%d")
+        ihsg = (
+            yfi.download("^JKSE", start=start_date, end=end_date, progress=False)
+            .dropna()
+        )[start_date:end_date]
+        print(f"New Data IHSG {start_date}-{end_date}\n", len(ihsg), " rows\t")
+        if len(ihsg) > 0:
+            print(ihsg)
+            ihsg = (
+                ihsg.assign(
+                    Kode="IHSG",
+                    time_updated = datetime.datetime.now(),
+                )
+                .reset_index()
+            )[["Date", "Kode", "Open", "High", "Low", "Close", "Volume", "time_updated"]]
+            ihsg.to_sql("historical", if_exists="append", con=con, index=False)
+            tickers = pd.read_sql(
+            """
+                SELECT DISTINCT Kode FROM historical
+                WHERE Kode != "IHSG"
+            """,
+            con=con,
+            ).iloc[:,0].to_list()
+            print("UPDATING historical TABLE..")
+            for i in range(0, len(tickers), 50):
+                ticker = [f"{kode}.JK" for kode in tickers[i : i + 50]]
+                df = (
+                    yfi.download(ticker, start=start_date, end=end_date, progress=False)
+                    .T.unstack(level=1)
+                    .T.reset_index()
+                    .dropna()
+                    .rename(columns={"level_1": "Kode"})
+                )[["Date", "Kode", "Open", "High", "Low", "Close", "Volume"]]
+                df["Kode"] = df["Kode"].str.replace(".JK", "")
+                df = df.assign(time_updated = datetime.datetime.now())
+                df.to_sql("historical", if_exists="append", con=con, index=False)
+
+
+            # update patterns database
+            tickers = ["IHSG"] + tickers
+            start = time.time()
+            for i, kode in enumerate(tickers):
+                print(f"Finding Patterns for {kode} #{i+1}\t\t time elapsed = {time.time() - start:.2f} s")
+                try:
+                    search_result = find_patterns(df=pd.read_sql(f"""
+                        SELECT * 
+                        FROM historical 
+                        WHERE Kode = '{kode}' 
+                        ORDER BY Date
+                    """,
+                    con=con,
+                    ))
+                    if i == 0:
+                        search_result.to_sql("patterns", if_exists="replace", con=con, index=False)
+                    else:
+                        search_result.to_sql("patterns", if_exists="append", con=con, index=False)
+                except:
+                    pass

ta-lib-0.4.0-src.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff41efcb1c011a4b4b6dfc91610b06e39b1d7973ed5d4dee55029a0ac4dc651
+size 1330299