{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/222734361","pull_request_review_id":161693535,"id":222734361,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyMjczNDM2MQ==","diff_hunk":"@@ -0,0 +1,906 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+var blockUICount = 0;\n+\n+$(document).ajaxStop(function () {\n+    if (blockUICount == 0) {\n+        $.unblockUI();\n+        blockUICount++;\n+    }\n+});\n+\n+$(document).ajaxStart(function () {\n+    if (blockUICount == 0) {\n+        $.blockUI({message: '<h3>Loading Stage Page...</h3>'});\n+    }\n+});\n+\n+$.extend( $.fn.dataTable.ext.type.order, {\n+    \"duration-pre\": ConvertDurationString,\n+\n+    \"duration-asc\": function ( a, b ) {\n+        a = ConvertDurationString( a );\n+        b = ConvertDurationString( b );\n+        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n+    },\n+\n+    \"duration-desc\": function ( a, b ) {\n+        a = ConvertDurationString( a );\n+        b = ConvertDurationString( b );\n+        return ((a < b) ? 1 : ((a > b) ? -1 : 0));\n+    }\n+} );\n+\n+// This function will only parse the URL under certain format\n+// e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1\n+// e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1\n+function stageEndPoint(appId) {\n+    var urlRegex = /https\\:\\/\\/[^\\/]+\\/([^\\/]+)\\/([^\\/]+)\\/([^\\/]+)?\\/?([^\\/]+)?\\/?([^\\/]+)?\\/?([^\\/]+)?/gm;\n+    var urlArray = urlRegex.exec(document.baseURI);\n+    var ind = urlArray.indexOf(\"proxy\");\n+    var queryString = document.baseURI.split('?');\n+    var words = document.baseURI.split('/');\n+    if (ind > 0) {\n+        var appId = urlArray[2];\n+        var stageId = queryString[1].split(\"&\").filter(word => word.includes(\"id=\"))[0].split(\"=\")[1];\n+        var indexOfProxy = words.indexOf(\"proxy\");\n+        var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');\n+        return newBaseURI + \"/api/v1/applications/\" + appId + \"/stages/\" + stageId;\n+    }\n+    ind = urlArray.indexOf(\"history\");\n+    if (ind > 0) {\n+        var appId = urlArray[2];\n+        var appAttemptId = urlArray[ind + 2];\n+        var stageId = queryString[1].split(\"&\").filter(word => word.includes(\"id=\"))[0].split(\"=\")[1];\n+        var indexOfHistory = words.indexOf(\"history\");\n+        var newBaseURI = words.slice(0, indexOfHistory).join('/');\n+        if (isNaN(appAttemptId) || appAttemptId == \"0\") {\n+            return newBaseURI + \"/api/v1/applications/\" + appId + \"/stages/\" + stageId;\n+        } else {\n+            return newBaseURI + \"/api/v1/applications/\" + appId + \"/\" + appAttemptId + \"/stages/\" + stageId;\n+        }\n+    }\n+    var stageId = queryString[1].split(\"&\").filter(word => word.includes(\"id=\"))[0].split(\"=\")[1];\n+    return location.origin + \"/api/v1/applications/\" + appId + \"/stages/\" + stageId;\n+}\n+\n+function getColumnNameForTaskMetricSummary(columnKey) {\n+    switch(columnKey) {\n+        case \"executorRunTime\":\n+            return \"Duration\";\n+            break;\n+\n+        case \"jvmGcTime\":\n+            return \"GC Time\";\n+            break;\n+\n+        case \"gettingResultTime\":\n+            return \"Getting Result Time\";\n+            break;\n+\n+        case \"inputMetrics\":\n+            return \"Input Size / Records\";\n+            break;\n+\n+        case \"outputMetrics\":\n+            return \"Output Size / Records\";\n+            break;\n+\n+        case \"peakExecutionMemory\":\n+            return \"Peak Execution Memory\";\n+            break;\n+\n+        case \"resultSerializationTime\":\n+            return \"Result Serialization Time\";\n+            break;\n+\n+        case \"schedulerDelay\":\n+            return \"Scheduler Delay\";\n+            break;\n+\n+        case \"diskBytesSpilled\":\n+            return \"Shuffle spill (disk)\";\n+            break;\n+\n+        case \"memoryBytesSpilled\":\n+            return \"Shuffle spill (memory)\";\n+            break;\n+\n+        case \"shuffleReadMetrics\":\n+            return \"Shuffle Read Size / Records\";\n+            break;\n+\n+        case \"shuffleWriteMetrics\":\n+            return \"Shuffle Write Size / Records\";\n+            break;\n+\n+        case \"executorDeserializeTime\":\n+            return \"Task Deserialization Time\";\n+            break;\n+\n+        default:\n+            return \"NA\";\n+    }\n+}\n+\n+function displayRowsForSummaryMetricsTable(row, type, columnIndex) {\n+    switch(row.metric) {\n+        case 'Input Size / Records':\n+            var str = formatBytes(row.data.bytesRead[columnIndex], type) + \" / \" +\n+              row.data.recordsRead[columnIndex];\n+            return str;\n+            break;\n+\n+        case 'Output Size / Records':\n+            var str = formatBytes(row.data.bytesWritten[columnIndex], type) + \" / \" +\n+              row.data.recordsWritten[columnIndex];\n+            return str;\n+            break;\n+\n+        case 'Shuffle Read Size / Records':\n+            var str = formatBytes(row.data.readBytes[columnIndex], type) + \" / \" +\n+              row.data.readRecords[columnIndex];\n+            return str;\n+            break;\n+\n+        case 'Shuffle Read Blocked Time':\n+            var str = formatDuration(row.data.fetchWaitTime[columnIndex]);\n+            return str;\n+            break;\n+\n+        case 'Shuffle Remote Reads':\n+            var str = formatBytes(row.data.remoteBytesRead[columnIndex], type);\n+            return str;\n+            break;\n+\n+        case 'Shuffle Write Size / Records':\n+            var str = formatBytes(row.data.writeBytes[columnIndex], type) + \" / \" +\n+              row.data.writeRecords[columnIndex];\n+            return str;\n+            break;\n+\n+        default:\n+            return (row.metric == 'Peak Execution Memory' || row.metric == 'Shuffle spill (memory)'\n+                    || row.metric == 'Shuffle spill (disk)') ? formatBytes(row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));\n+\n+    }\n+}\n+\n+function createDataTableForTaskSummaryMetricsTable(task_summary_metrics_table) {\n+    var taskMetricsTable = \"#summary-metrics-table\";\n+    if ($.fn.dataTable.isDataTable(taskMetricsTable)) {\n+        taskSummaryMetricsDataTable.clear().draw();\n+        taskSummaryMetricsDataTable.rows.add(task_summary_metrics_table).draw();\n+    } else {\n+        var task_conf = {\n+            \"data\": task_summary_metrics_table,\n+            \"columns\": [\n+                {data : 'metric'},\n+                // Min\n+                {\n+                    data: function (row, type) {\n+                        return displayRowsForSummaryMetricsTable(row, type, 0);\n+                    }\n+                },\n+                // 25th percentile\n+                {\n+                    data: function (row, type) {\n+                        return displayRowsForSummaryMetricsTable(row, type, 1);\n+                    }\n+                },\n+                // Median\n+                {\n+                    data: function (row, type) {\n+                        return displayRowsForSummaryMetricsTable(row, type, 2);\n+                    }\n+                },\n+                // 75th percentile\n+                {\n+                    data: function (row, type) {\n+                        return displayRowsForSummaryMetricsTable(row, type, 3);\n+                    }\n+                },\n+                // Max\n+                {\n+                    data: function (row, type) {\n+                        return displayRowsForSummaryMetricsTable(row, type, 4);\n+                    }\n+                }\n+            ],\n+            \"columnDefs\": [\n+                { \"type\": \"duration\", \"targets\": 1 },\n+                { \"type\": \"duration\", \"targets\": 2 },\n+                { \"type\": \"duration\", \"targets\": 3 },\n+                { \"type\": \"duration\", \"targets\": 4 },\n+                { \"type\": \"duration\", \"targets\": 5 }\n+            ],\n+            \"paging\": false,\n+            \"searching\": false,\n+            \"order\": [[0, \"asc\"]]\n+        };\n+        taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(task_conf);\n+    }\n+    task_summary_metrics_table_current_state_array = task_summary_metrics_table.slice();\n+}\n+\n+var task_summary_metrics_table_array = [];\n+var task_summary_metrics_table_current_state_array = [];\n+var taskSummaryMetricsDataTable;\n+\n+$(document).ready(function () {\n+    setDataTableDefaults();\n+\n+    $(\"#showAdditionalMetrics\").append(\n+        \"<div><a id='additionalMetrics'>\" +\n+        \"<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>\" +\n+        \" Show Additional Metrics\" +\n+        \"</a></div>\" +\n+        \"<div class='container-fluid container-fluid-div' id='toggle-metrics' hidden>\" +\n+        \"<div><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>\" +\n+        \"<div id='scheduler_delay'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>\" +\n+        \"<div id='task_deserialization_time'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Task Deserialization Time</div>\" +\n+        \"<div id='shuffle_read_blocked_time'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked Time</div>\" +\n+        \"<div id='shuffle_remote_reads'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>\" +\n+        \"<div id='result_serialization_time'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Result Serialization Time</div>\" +\n+        \"<div id='getting_result_time'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>\" +\n+        \"<div id='peak_execution_memory'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>\" +\n+        \"</div>\");\n+\n+    $('#scheduler_delay').attr(\"data-toggle\", \"tooltip\")\n+        .attr(\"data-placement\", \"bottom\")\n+        .attr(\"title\", \"Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send \" +\n+            \"the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.\");\n+    $('#task_deserialization_time').attr(\"data-toggle\", \"tooltip\")\n+        .attr(\"data-placement\", \"bottom\")\n+        .attr(\"title\", \"Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.\");\n+    $('#shuffle_read_blocked_time').attr(\"data-toggle\", \"tooltip\")\n+        .attr(\"data-placement\", \"bottom\")\n+        .attr(\"title\", \"Time that the task spent blocked waiting for shuffle data to be read from remote machines.\");\n+    $('#shuffle_remote_reads').attr(\"data-toggle\", \"tooltip\")\n+        .attr(\"data-placement\", \"bottom\")\n+        .attr(\"title\", \"Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. \");\n+    $('#result_serialization_time').attr(\"data-toggle\", \"tooltip\")\n+            .attr(\"data-placement\", \"bottom\")\n+            .attr(\"title\", \"Time spent serializing the task result on the executor before sending it back to the driver.\");\n+    $('#getting_result_time').attr(\"data-toggle\", \"tooltip\")\n+            .attr(\"data-placement\", \"bottom\")\n+            .attr(\"title\", \"Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.\");\n+    $('#peak_execution_memory').attr(\"data-toggle\", \"tooltip\")\n+            .attr(\"data-placement\", \"bottom\")\n+            .attr(\"title\", \"Execution memory refers to the memory used by internal data structures created during \" +\n+                \"shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator \" +\n+                \"should be approximately the sum of the peak sizes across all such data structures created \" +\n+                \"in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and \" +\n+                \"external sort.\");\n+    $('#scheduler_delay').tooltip(true);\n+    $('#task_deserialization_time').tooltip(true);\n+    $('#shuffle_read_blocked_time').tooltip(true);\n+    $('#shuffle_remote_reads').tooltip(true);\n+    $('#result_serialization_time').tooltip(true);\n+    $('#getting_result_time').tooltip(true);\n+    $('#peak_execution_memory').tooltip(true);\n+    tasksSummary = $(\"#parent-container\");\n+    getStandAloneAppId(function (appId) {\n+\n+        var endPoint = stageEndPoint(appId);\n+        $.getJSON(endPoint, function(response, status, jqXHR) {\n+\n+            var stageAttemptId = getStageAttemptId();\n+            var responseBody = response[stageAttemptId];\n+            var dataToShow = {};\n+            dataToShow.showInputData = responseBody.inputBytes > 0?true:false;\n+            dataToShow.showOutputData = responseBody.outputBytes > 0?true:false;\n+            dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0?true:false;\n+            dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0?true:false;\n+            dataToShow.showBytesSpilledData =\n+                (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0)?true:false;\n+\n+            // prepare data for task aggregated metrics table\n+            indices = Object.keys(responseBody.executorSummary);\n+            var executor_summary_table = [];\n+            indices.forEach(function (ix) {\n+               responseBody.executorSummary[ix].id = ix;\n+               executor_summary_table.push(responseBody.executorSummary[ix]);\n+            });\n+\n+            // prepare data for accumulatorUpdates\n+            var accumulator_table = responseBody.accumulatorUpdates.filter(accumUpdate =>\n+                !(accumUpdate.name).toString().includes(\"internal.\"));\n+\n+            // rendering the UI page\n+            var data = {\"executors\": response};\n+            $.get(createTemplateURI(appId, \"stagespage\"), function(template) {\n+                tasksSummary.append(Mustache.render($(template).filter(\"#stages-summary-template\").html(), data));\n+\n+                $(\"#additionalMetrics\").click(function(){\n+                    $(\"#arrowtoggle1\").toggleClass(\"arrow-open arrow-closed\");\n+                    $(\"#toggle-metrics\").toggle();\n+                });\n+\n+                $(\"#aggregatedMetrics\").click(function(){\n+                    $(\"#arrowtoggle2\").toggleClass(\"arrow-open arrow-closed\");\n+                    $(\"#toggle-aggregatedMetrics\").toggle();\n+                });\n+\n+                var quantiles = \"0,0.25,0.5,0.75,1.0\";\n+                $.getJSON(stageEndPoint(appId) + \"/\"+stageAttemptId+\"/taskSummary?quantiles=\"+quantiles,\n+                  function(taskMetricsResponse, status, jqXHR) {\n+                    var taskMetricKeys = Object.keys(taskMetricsResponse);\n+                    taskMetricKeys.forEach(function (ix) {\n+                        var columnName = getColumnNameForTaskMetricSummary(ix);\n+                        if (columnName == \"Shuffle Read Size / Records\") {\n+                            var row1 = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 3\n+                            };\n+                            var row2 = {\n+                                \"metric\": \"Shuffle Read Blocked Time\",\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 13\n+                            };\n+                            var row3 = {\n+                                \"metric\": \"Shuffle Remote Reads\",\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 14\n+                            };\n+                            if (dataToShow.showShuffleReadData) {\n+                                task_summary_metrics_table_array.push(row1);\n+                            }\n+                            task_summary_metrics_table_array.push(row2);\n+                            task_summary_metrics_table_array.push(row3);\n+                        }\n+                        else if (columnName == \"Scheduler Delay\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 11\n+                            };\n+                            task_summary_metrics_table_array.push(row);\n+                        }\n+                        else if (columnName == \"Task Deserialization Time\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 12\n+                            };\n+                            task_summary_metrics_table_array.push(row);\n+                        }\n+                        else if (columnName == \"Result Serialization Time\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 15\n+                            };\n+                            task_summary_metrics_table_array.push(row);\n+                        }\n+                        else if (columnName == \"Getting Result Time\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 16\n+                            };\n+                            task_summary_metrics_table_array.push(row);\n+                        }\n+                        else if (columnName == \"Peak Execution Memory\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 17\n+                            };\n+                            task_summary_metrics_table_array.push(row);\n+                        }\n+                        else if (columnName == \"Input Size / Records\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 1\n+                            };\n+                            if (dataToShow.showInputData) {\n+                                task_summary_metrics_table_array.push(row);\n+                            }\n+                        }\n+                        else if (columnName == \"Output Size / Records\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 2\n+                            };\n+                            if (dataToShow.showOutputData) {\n+                                task_summary_metrics_table_array.push(row);\n+                            }\n+                        }\n+                        else if (columnName == \"Shuffle Write Size / Records\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 4\n+                            };\n+                            if (dataToShow.showShuffleWriteData) {\n+                                task_summary_metrics_table_array.push(row);\n+                            }\n+                        }\n+                        else if (columnName == \"Shuffle spill (disk)\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 5\n+                            };\n+                            if (dataToShow.showBytesSpilledData) {\n+                                task_summary_metrics_table_array.push(row);\n+                            }\n+                        }\n+                        else if (columnName == \"Shuffle spill (memory)\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 6\n+                            };\n+                            if (dataToShow.showBytesSpilledData) {\n+                                task_summary_metrics_table_array.push(row);\n+                            }\n+                        }\n+                        else if (columnName != \"NA\") {\n+                            var row = {\n+                                \"metric\": columnName,\n+                                \"data\": taskMetricsResponse[ix],\n+                                \"checkboxId\": 0\n+                            };\n+                            task_summary_metrics_table_array.push(row);\n+                        }\n+                    });\n+                    var task_summary_metrics_table_filtered_array =\n+                        task_summary_metrics_table_array.filter(row => row.checkboxId < 11);\n+                    createDataTableForTaskSummaryMetricsTable(task_summary_metrics_table_filtered_array);\n+                });\n+\n+                // building task aggregated metrics by executor table\n+                var executorSummaryTable = \"#summary-executor-table\";\n+                var executor_summary_conf = {\n+                    \"data\": executor_summary_table,\n+                    \"columns\": [\n+                        {data : \"id\"},\n+                        {data : \"executorLogs\", render: formatLogsCells},\n+                        {data : \"hostPort\"},\n+                        {\n+                            data : function (row, type) {\n+                                return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;\n+                            }\n+                        },\n+                        {\n+                            data : function (row, type) {\n+                                var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;\n+                                return type === 'display' ? totaltasks : totaltasks.toString();\n+                            }\n+                        },\n+                        {data : \"failedTasks\"},\n+                        {data : \"killedTasks\"},\n+                        {data : \"succeededTasks\"},\n+                        {data : \"isBlacklistedForStage\"},\n+                        {\n+                            data : function (row, type) {\n+                                return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + \" / \" + row.inputRecords : 0;\n+                            }\n+                        },\n+                        {\n+                            data : function (row, type) {\n+                                return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + \" / \" + row.outputRecords : 0;\n+                            }\n+                        },\n+                        {\n+                            data : function (row, type) {\n+                                return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + \" / \" + row.shuffleReadRecords : 0;\n+                            }\n+                        },\n+                        {\n+                            data : function (row, type) {\n+                                return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + \" / \" + row.shuffleWriteRecords : 0;\n+                            }\n+                        },\n+                        {\n+                            data : function (row, type) {\n+                                return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : \"\";\n+                            }\n+                        },\n+                        {\n+                            data : function (row, type) {\n+                                return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : \"\";\n+                            }\n+                        }\n+                    ],\n+                    \"order\": [[0, \"asc\"]]\n+                }\n+                var executorSummaryTableSelector =\n+                  $(executorSummaryTable).DataTable(executor_summary_conf);\n+                $('#parent-container [data-toggle=\"tooltip\"]').tooltip();\n+\n+                // building accumulator update table\n+                var accumulatorTable = \"#accumulator-table\";\n+                var accumulator_conf = {\n+                    \"data\": accumulator_table,\n+                    \"columns\": [\n+                        {data : \"id\"},\n+                        {data : \"name\"},\n+                        {data : \"value\"}\n+                    ],\n+                    \"paging\": false,\n+                    \"searching\": false,\n+                    \"order\": [[0, \"asc\"]]\n+                }\n+                $(accumulatorTable).DataTable(accumulator_conf);\n+\n+                // building tasks table that uses server side functionality\n+                var taskTable = \"#active-tasks-table\";\n+                var task_conf = {\n+                    \"serverSide\": true,\n+                    \"paging\": true,\n+                    \"info\": true,\n+                    \"processing\": true,\n+                    \"lengthMenu\": [[20, 40, 60, 100, responseBody.numTasks], [20, 40, 60, 100, \"All\"]],\n+                    \"ajax\": {\n+                        \"url\": stageEndPoint(appId) + \"/\" + stageAttemptId + \"/taskTable\",\n+                        \"data\": {\n+                            \"numTasks\": responseBody.numTasks\n+                        },\n+                        \"dataSrc\": function ( jsons ) {\n+                            var jsonStr = JSON.stringify(jsons);\n+                            var tasksToShow = JSON.parse(jsonStr);\n+                            return tasksToShow.aaData;\n+                        }\n+                    },\n+                    \"columns\": [\n+                        {data: function (row, type) {\n+                            return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;\n+                            },\n+                            name: \"Index\"\n+                        },\n+                        {data : \"taskId\", name: \"ID\"},\n+                        {data : \"attempt\", name: \"Attempt\"},\n+                        {data : \"status\", name: \"Status\"},\n+                        {data : \"taskLocality\", name: \"Locality Level\"},","path":"core/src/main/resources/org/apache/spark/ui/static/stagepage.js","position":542,"original_position":575,"commit_id":"8cf0e3467b4ad89d5757f08521f368d5da5c97af","original_commit_id":"930a261619cfb39dd7d87027a1af2788c22cb44a","user":{"login":"pgandhi999","id":22228190,"node_id":"MDQ6VXNlcjIyMjI4MTkw","avatar_url":"https://avatars0.githubusercontent.com/u/22228190?v=4","gravatar_id":"","url":"https://api.github.com/users/pgandhi999","html_url":"https://github.com/pgandhi999","followers_url":"https://api.github.com/users/pgandhi999/followers","following_url":"https://api.github.com/users/pgandhi999/following{/other_user}","gists_url":"https://api.github.com/users/pgandhi999/gists{/gist_id}","starred_url":"https://api.github.com/users/pgandhi999/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pgandhi999/subscriptions","organizations_url":"https://api.github.com/users/pgandhi999/orgs","repos_url":"https://api.github.com/users/pgandhi999/repos","events_url":"https://api.github.com/users/pgandhi999/events{/privacy}","received_events_url":"https://api.github.com/users/pgandhi999/received_events","type":"User","site_admin":false},"body":"As discussed, I am using existing sorting API's so I have to explicitly write code to sort it the way described above. If it is nit for now, can do it as part of another PR.","created_at":"2018-10-04T16:12:02Z","updated_at":"2018-10-04T16:12:02Z","html_url":"https://github.com/apache/spark/pull/21688#discussion_r222734361","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/21688","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/222734361"},"html":{"href":"https://github.com/apache/spark/pull/21688#discussion_r222734361"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/21688"}},"in_reply_to_id":221672381},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/21688","id":198538814,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk4NTM4ODE0","html_url":"https://github.com/apache/spark/pull/21688","diff_url":"https://github.com/apache/spark/pull/21688.diff","patch_url":"https://github.com/apache/spark/pull/21688.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/21688","number":21688,"state":"open","locked":false,"title":"[SPARK-21809] : Change Stage Page to use datatables to support sorting columns and searching","user":{"login":"pgandhi999","id":22228190,"node_id":"MDQ6VXNlcjIyMjI4MTkw","avatar_url":"https://avatars0.githubusercontent.com/u/22228190?v=4","gravatar_id":"","url":"https://api.github.com/users/pgandhi999","html_url":"https://github.com/pgandhi999","followers_url":"https://api.github.com/users/pgandhi999/followers","following_url":"https://api.github.com/users/pgandhi999/following{/other_user}","gists_url":"https://api.github.com/users/pgandhi999/gists{/gist_id}","starred_url":"https://api.github.com/users/pgandhi999/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pgandhi999/subscriptions","organizations_url":"https://api.github.com/users/pgandhi999/orgs","repos_url":"https://api.github.com/users/pgandhi999/repos","events_url":"https://api.github.com/users/pgandhi999/events{/privacy}","received_events_url":"https://api.github.com/users/pgandhi999/received_events","type":"User","site_admin":false},"body":"Support column sort, pagination and search for Stage Page using jQuery DataTable and REST API. Before this commit, the Stage page generated a hard-coded HTML table that could not support search. Supporting search and sort (over all applications rather than the 20 entries in the current page) in any case will greatly improve the user experience.\r\nCreated the stagespage-template.html for displaying application information in datables. Added REST api endpoint and javascript code to fetch data from the endpoint and display it on the data table.\r\nBecause of the above change, certain functionalities in the page had to be modified to support the addition of datatables. For example, the toggle checkbox 'Select All' previously would add the checked fields as columns in the Task table and as rows in the Summary Metrics table, but after the change, only columns are added in the Task Table as it got tricky to add rows dynamically in the datatables.\r\n\r\n## How was this patch tested?\r\nI have attached the screenshots of the Stage Page UI before and after the fix.\r\n**Before:**\r\n\r\n<img width=\"1419\" alt=\"30564304-35991e1c-9c8a-11e7-850f-2ac7a347f600\" src=\"https://user-images.githubusercontent.com/22228190/42137915-52054558-7d3a-11e8-8c85-433b2c94161d.png\">\r\n\r\n<img width=\"1435\" alt=\"31360592-cbaa2bae-ad14-11e7-941d-95b4c7d14970\" src=\"https://user-images.githubusercontent.com/22228190/42137928-79df500a-7d3a-11e8-9068-5630afe46ff3.png\">\r\n\r\n**After:**\r\n\r\n<img width=\"1432\" alt=\"31360591-c5650ee4-ad14-11e7-9665-5a08d8f21830\" src=\"https://user-images.githubusercontent.com/22228190/42137936-a3fb9f42-7d3a-11e8-8502-22b3897cbf64.png\">\r\n\r\n<img width=\"1388\" alt=\"31360604-d266b6b0-ad14-11e7-94b5-dcc4bb5443f4\" src=\"https://user-images.githubusercontent.com/22228190/42137970-0fabc58c-7d3b-11e8-95ad-383b1bd1f106.png\">\r\n\r\n\r\n","created_at":"2018-07-01T19:28:58Z","updated_at":"2018-10-04T16:12:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"90443b2dfc0c5374c16b10cceadfaf850d522144","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/21688/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/21688/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/21688/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/8cf0e3467b4ad89d5757f08521f368d5da5c97af","head":{"label":"pgandhi999:SPARK-21809-2.3","ref":"SPARK-21809-2.3","sha":"8cf0e3467b4ad89d5757f08521f368d5da5c97af","user":{"login":"pgandhi999","id":22228190,"node_id":"MDQ6VXNlcjIyMjI4MTkw","avatar_url":"https://avatars0.githubusercontent.com/u/22228190?v=4","gravatar_id":"","url":"https://api.github.com/users/pgandhi999","html_url":"https://github.com/pgandhi999","followers_url":"https://api.github.com/users/pgandhi999/followers","following_url":"https://api.github.com/users/pgandhi999/following{/other_user}","gists_url":"https://api.github.com/users/pgandhi999/gists{/gist_id}","starred_url":"https://api.github.com/users/pgandhi999/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pgandhi999/subscriptions","organizations_url":"https://api.github.com/users/pgandhi999/orgs","repos_url":"https://api.github.com/users/pgandhi999/repos","events_url":"https://api.github.com/users/pgandhi999/events{/privacy}","received_events_url":"https://api.github.com/users/pgandhi999/received_events","type":"User","site_admin":false},"repo":{"id":97985305,"node_id":"MDEwOlJlcG9zaXRvcnk5Nzk4NTMwNQ==","name":"spark","full_name":"pgandhi999/spark","private":false,"owner":{"login":"pgandhi999","id":22228190,"node_id":"MDQ6VXNlcjIyMjI4MTkw","avatar_url":"https://avatars0.githubusercontent.com/u/22228190?v=4","gravatar_id":"","url":"https://api.github.com/users/pgandhi999","html_url":"https://github.com/pgandhi999","followers_url":"https://api.github.com/users/pgandhi999/followers","following_url":"https://api.github.com/users/pgandhi999/following{/other_user}","gists_url":"https://api.github.com/users/pgandhi999/gists{/gist_id}","starred_url":"https://api.github.com/users/pgandhi999/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pgandhi999/subscriptions","organizations_url":"https://api.github.com/users/pgandhi999/orgs","repos_url":"https://api.github.com/users/pgandhi999/repos","events_url":"https://api.github.com/users/pgandhi999/events{/privacy}","received_events_url":"https://api.github.com/users/pgandhi999/received_events","type":"User","site_admin":false},"html_url":"https://github.com/pgandhi999/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/pgandhi999/spark","forks_url":"https://api.github.com/repos/pgandhi999/spark/forks","keys_url":"https://api.github.com/repos/pgandhi999/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pgandhi999/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pgandhi999/spark/teams","hooks_url":"https://api.github.com/repos/pgandhi999/spark/hooks","issue_events_url":"https://api.github.com/repos/pgandhi999/spark/issues/events{/number}","events_url":"https://api.github.com/repos/pgandhi999/spark/events","assignees_url":"https://api.github.com/repos/pgandhi999/spark/assignees{/user}","branches_url":"https://api.github.com/repos/pgandhi999/spark/branches{/branch}","tags_url":"https://api.github.com/repos/pgandhi999/spark/tags","blobs_url":"https://api.github.com/repos/pgandhi999/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pgandhi999/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pgandhi999/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/pgandhi999/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pgandhi999/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/pgandhi999/spark/languages","stargazers_url":"https://api.github.com/repos/pgandhi999/spark/stargazers","contributors_url":"https://api.github.com/repos/pgandhi999/spark/contributors","subscribers_url":"https://api.github.com/repos/pgandhi999/spark/subscribers","subscription_url":"https://api.github.com/repos/pgandhi999/spark/subscription","commits_url":"https://api.github.com/repos/pgandhi999/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/pgandhi999/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/pgandhi999/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/pgandhi999/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/pgandhi999/spark/contents/{+path}","compare_url":"https://api.github.com/repos/pgandhi999/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pgandhi999/spark/merges","archive_url":"https://api.github.com/repos/pgandhi999/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pgandhi999/spark/downloads","issues_url":"https://api.github.com/repos/pgandhi999/spark/issues{/number}","pulls_url":"https://api.github.com/repos/pgandhi999/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/pgandhi999/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/pgandhi999/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pgandhi999/spark/labels{/name}","releases_url":"https://api.github.com/repos/pgandhi999/spark/releases{/id}","deployments_url":"https://api.github.com/repos/pgandhi999/spark/deployments","created_at":"2017-07-21T20:46:46Z","updated_at":"2018-10-03T21:34:09Z","pushed_at":"2018-10-03T22:03:39Z","git_url":"git://github.com/pgandhi999/spark.git","ssh_url":"git@github.com:pgandhi999/spark.git","clone_url":"https://github.com/pgandhi999/spark.git","svn_url":"https://github.com/pgandhi999/spark","homepage":null,"size":281674,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"d7ae36a810bfcbedfe7360eb2cdbbc3ca970e4d0","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"node_id":"MDEwOlJlcG9zaXRvcnkxNzE2NTY1OA==","name":"spark","full_name":"apache/spark","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2018-10-04T12:20:30Z","pushed_at":"2018-10-04T16:06:15Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":283610,"stargazers_count":18979,"watchers_count":18979,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":17061,"mirror_url":null,"archived":false,"open_issues_count":519,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":17061,"open_issues":519,"watchers":18979,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/21688"},"html":{"href":"https://github.com/apache/spark/pull/21688"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/21688"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/21688/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/21688/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/21688/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/8cf0e3467b4ad89d5757f08521f368d5da5c97af"}},"author_association":"NONE"}}
{"actor":{"display_login":"pgandhi999"}}