{"action":"created","comment":{"url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/comments/63505895","id":63505895,"diff_hunk":"@@ -489,119 +488,71 @@ private XmlDocument LoadContentFromDatabase()\n             {\r\n                 // Try to log to the DB\r\n                 LogHelper.Info<content>(\"Loading content from database...\");\r\n-\r\n-                var hierarchy = new Dictionary<int, List<int>>();\r\n-                var nodeIndex = new Dictionary<int, XmlNode>();\r\n-\r\n+                \r\n                 try\r\n                 {\r\n                     LogHelper.Debug<content>(\"Republishing starting\");\r\n \r\n                     lock (DbReadSyncLock)\r\n                     {\r\n+                        //TODO: This is what we should do , but converting to use XDocument would be breaking unless we convert\r\n+                        // to XmlDocument at the end of this, but again, this would be bad for memory... though still not nearly as\r\n+                        // bad as what is happening before!\r\n+                        // We'll keep using XmlDocument for now though, but XDocument xml generation is much faster:\r\n+                        // https://blogs.msdn.microsoft.com/codejunkie/2008/10/08/xmldocument-vs-xelement-performance/\r\n+                        // I think we already have code in here to convert XDocument to XmlDocument but in case we don't here\r\n+                        // it is: https://blogs.msdn.microsoft.com/marcelolr/2009/03/13/fast-way-to-convert-xmldocument-into-xdocument/\r\n+\r\n+                        //// Prepare an XmlDocument with an appropriate inline DTD to match\r\n+                        //// the expected content\r\n+                        //var parent = new XElement(\"root\", new XAttribute(\"id\", \"-1\"));\r\n+                        //var xmlDoc = new XDocument(\r\n+                        //    new XDocumentType(\"root\", null, null, DocumentType.GenerateDtd()),\r\n+                        //    parent);\r\n \r\n-                        // Lets cache the DTD to save on the DB hit on the subsequent use\r\n-                        string dtd = DocumentType.GenerateDtd();\r\n-\r\n-                        // Prepare an XmlDocument with an appropriate inline DTD to match\r\n-                        // the expected content\r\n                         var xmlDoc = new XmlDocument();\r\n-                        InitializeXml(xmlDoc, dtd);\r\n+                        var doctype = xmlDoc.CreateDocumentType(\"root\", null, null, \r\n+                            ApplicationContext.Current.Services.ContentTypeService.GetContentTypesDtd());\r\n+                        xmlDoc.AppendChild(doctype);\r\n+                        var parent = xmlDoc.CreateElement(\"root\");\r\n+                        var pIdAtt = xmlDoc.CreateAttribute(\"id\");\r\n+                        pIdAtt.Value = \"-1\";\r\n+                        parent.Attributes.Append(pIdAtt);\r\n+                        xmlDoc.AppendChild(parent);\r\n \r\n                         // Esben Carlsen: At some point we really need to put all data access into to a tier of its own.\r\n                         // CLN - added checks that document xml is for a document that is actually published.\r\n-                        string sql =\r\n-                            @\"select umbracoNode.id, umbracoNode.parentId, umbracoNode.sortOrder, cmsContentXml.xml from umbracoNode \r\n+                        const string sql = @\"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.xml, umbracoNode.level from umbracoNode\r\n inner join cmsContentXml on cmsContentXml.nodeId = umbracoNode.id and umbracoNode.nodeObjectType = @type\r\n where umbracoNode.id in (select cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1)\r\n-order by umbracoNode.level, umbracoNode.sortOrder\";\r\n-\r\n-\r\n+order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder\";\r\n \r\n-                        using (\r\n-                            IRecordsReader dr = SqlHelper.ExecuteReader(sql,\r\n-                                                                        SqlHelper.CreateParameter(\"@type\",\r\n-                                                                                                  new Guid(\r\n-                                                                                                      Constants.ObjectTypes.Document)))\r\n-                            )\r\n+                        XmlElement last = null;\r\n+                        var db = ApplicationContext.Current.DatabaseContext.Database;\r\n+                        //NOTE: Query creates a reader - does not load all into memory\r\n+                        foreach (var row in db.Query<dynamic>(sql, new { type  = new Guid(Constants.ObjectTypes.Document)}))\r\n                         {\r\n-                            while (dr.Read())\r\n+                            string parentId = ((int)row.parentID).ToInvariantString();\r\n+                            string xml = row.xml;\r\n+\r\n+                            //if the parentid is changing\r\n+                            if (last != null && last.GetAttribute(\"parentID\") != parentId)\r\n                             {\r\n-                                int currentId = dr.GetInt(\"id\");\r\n-                                int parentId = dr.GetInt(\"parentId\");\r\n-                                string xml = dr.GetString(\"xml\");\r\n-\r\n-                                // fix sortOrder - see notes in UpdateSortOrder\r","path":"src/Umbraco.Web/umbraco.presentation/content.cs","position":125,"original_position":116,"commit_id":"512c19a13208ab7091c34174479fad10255b6f4a","original_commit_id":"0d11b15b3f716bdb90b75585a3cca2ba037dad55","user":{"login":"Shazwazza","id":1742685,"avatar_url":"https://avatars.githubusercontent.com/u/1742685?v=3","gravatar_id":"","url":"https://api.github.com/users/Shazwazza","html_url":"https://github.com/Shazwazza","followers_url":"https://api.github.com/users/Shazwazza/followers","following_url":"https://api.github.com/users/Shazwazza/following{/other_user}","gists_url":"https://api.github.com/users/Shazwazza/gists{/gist_id}","starred_url":"https://api.github.com/users/Shazwazza/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Shazwazza/subscriptions","organizations_url":"https://api.github.com/users/Shazwazza/orgs","repos_url":"https://api.github.com/users/Shazwazza/repos","events_url":"https://api.github.com/users/Shazwazza/events{/privacy}","received_events_url":"https://api.github.com/users/Shazwazza/received_events","type":"User","site_admin":false},"body":"I've pushed some changes to this branch now. We now ensure that the sortOrder is maintained here from what is stored in the db. I've then changed how we Clone the XML document so that it's only cloned once per requests (if there is a request available). This fixes what happens when we sort, before we would clone the XML file for every item that was involved in the sort ... that could be quite a lot which would result in huge memory usage if the xml content cache is large.","created_at":"2016-05-17T11:29:46Z","updated_at":"2016-05-17T11:29:46Z","html_url":"https://github.com/umbraco/Umbraco-CMS/pull/1255#discussion_r63505895","pull_request_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255","_links":{"self":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/comments/63505895"},"html":{"href":"https://github.com/umbraco/Umbraco-CMS/pull/1255#discussion_r63505895"},"pull_request":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255"}}},"pull_request":{"url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255","id":68873600,"html_url":"https://github.com/umbraco/Umbraco-CMS/pull/1255","diff_url":"https://github.com/umbraco/Umbraco-CMS/pull/1255.diff","patch_url":"https://github.com/umbraco/Umbraco-CMS/pull/1255.patch","issue_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/1255","number":1255,"state":"open","locked":false,"title":"This fixes the LoadContentFromDatabase method","user":{"login":"Shazwazza","id":1742685,"avatar_url":"https://avatars.githubusercontent.com/u/1742685?v=3","gravatar_id":"","url":"https://api.github.com/users/Shazwazza","html_url":"https://github.com/Shazwazza","followers_url":"https://api.github.com/users/Shazwazza/followers","following_url":"https://api.github.com/users/Shazwazza/following{/other_user}","gists_url":"https://api.github.com/users/Shazwazza/gists{/gist_id}","starred_url":"https://api.github.com/users/Shazwazza/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Shazwazza/subscriptions","organizations_url":"https://api.github.com/users/Shazwazza/orgs","repos_url":"https://api.github.com/users/Shazwazza/repos","events_url":"https://api.github.com/users/Shazwazza/events{/privacy}","received_events_url":"https://api.github.com/users/Shazwazza/received_events","type":"User","site_admin":false},"body":"before this was storing every single xml node in memory in dictionaries (x2) and then performing the re-organization of all nodes afterwords to construct a document, now we just read each row and organize the document accordingly,","created_at":"2016-05-04T14:42:39Z","updated_at":"2016-05-17T11:29:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5902246709baa122d344d0339530ae1c9cbd048e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255/commits","review_comments_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255/comments","review_comment_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/comments{/number}","comments_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/1255/comments","statuses_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/statuses/512c19a13208ab7091c34174479fad10255b6f4a","head":{"label":"umbraco:better-memory-xmlcache","ref":"better-memory-xmlcache","sha":"512c19a13208ab7091c34174479fad10255b6f4a","user":{"login":"umbraco","id":1419552,"avatar_url":"https://avatars.githubusercontent.com/u/1419552?v=3","gravatar_id":"","url":"https://api.github.com/users/umbraco","html_url":"https://github.com/umbraco","followers_url":"https://api.github.com/users/umbraco/followers","following_url":"https://api.github.com/users/umbraco/following{/other_user}","gists_url":"https://api.github.com/users/umbraco/gists{/gist_id}","starred_url":"https://api.github.com/users/umbraco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/umbraco/subscriptions","organizations_url":"https://api.github.com/users/umbraco/orgs","repos_url":"https://api.github.com/users/umbraco/repos","events_url":"https://api.github.com/users/umbraco/events{/privacy}","received_events_url":"https://api.github.com/users/umbraco/received_events","type":"Organization","site_admin":false},"repo":{"id":10601208,"name":"Umbraco-CMS","full_name":"umbraco/Umbraco-CMS","owner":{"login":"umbraco","id":1419552,"avatar_url":"https://avatars.githubusercontent.com/u/1419552?v=3","gravatar_id":"","url":"https://api.github.com/users/umbraco","html_url":"https://github.com/umbraco","followers_url":"https://api.github.com/users/umbraco/followers","following_url":"https://api.github.com/users/umbraco/following{/other_user}","gists_url":"https://api.github.com/users/umbraco/gists{/gist_id}","starred_url":"https://api.github.com/users/umbraco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/umbraco/subscriptions","organizations_url":"https://api.github.com/users/umbraco/orgs","repos_url":"https://api.github.com/users/umbraco/repos","events_url":"https://api.github.com/users/umbraco/events{/privacy}","received_events_url":"https://api.github.com/users/umbraco/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/umbraco/Umbraco-CMS","description":"The simple, flexible and friendly ASP.NET CMS used by more than 350.000 websites","fork":false,"url":"https://api.github.com/repos/umbraco/Umbraco-CMS","forks_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/forks","keys_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/keys{/key_id}","collaborators_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/teams","hooks_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/hooks","issue_events_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/events{/number}","events_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/events","assignees_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/assignees{/user}","branches_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/branches{/branch}","tags_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/tags","blobs_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/refs{/sha}","trees_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/trees{/sha}","statuses_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/statuses/{sha}","languages_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/languages","stargazers_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/stargazers","contributors_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/contributors","subscribers_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/subscribers","subscription_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/subscription","commits_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/commits{/sha}","git_commits_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/commits{/sha}","comments_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/comments{/number}","issue_comment_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/comments{/number}","contents_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/contents/{+path}","compare_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/compare/{base}...{head}","merges_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/merges","archive_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/downloads","issues_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues{/number}","pulls_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls{/number}","milestones_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/milestones{/number}","notifications_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/labels{/name}","releases_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/releases{/id}","deployments_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/deployments","created_at":"2013-06-10T13:50:46Z","updated_at":"2016-05-17T09:15:53Z","pushed_at":"2016-05-17T11:27:46Z","git_url":"git://github.com/umbraco/Umbraco-CMS.git","ssh_url":"git@github.com:umbraco/Umbraco-CMS.git","clone_url":"https://github.com/umbraco/Umbraco-CMS.git","svn_url":"https://github.com/umbraco/Umbraco-CMS","homepage":"http://umbraco.com","size":190152,"stargazers_count":1208,"watchers_count":1208,"language":"C#","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":973,"mirror_url":null,"open_issues_count":73,"forks":973,"open_issues":73,"watchers":1208,"default_branch":"dev-v7"}},"base":{"label":"umbraco:dev-v7","ref":"dev-v7","sha":"5f8747b4d3770ea2585a86f080a582f1826c3160","user":{"login":"umbraco","id":1419552,"avatar_url":"https://avatars.githubusercontent.com/u/1419552?v=3","gravatar_id":"","url":"https://api.github.com/users/umbraco","html_url":"https://github.com/umbraco","followers_url":"https://api.github.com/users/umbraco/followers","following_url":"https://api.github.com/users/umbraco/following{/other_user}","gists_url":"https://api.github.com/users/umbraco/gists{/gist_id}","starred_url":"https://api.github.com/users/umbraco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/umbraco/subscriptions","organizations_url":"https://api.github.com/users/umbraco/orgs","repos_url":"https://api.github.com/users/umbraco/repos","events_url":"https://api.github.com/users/umbraco/events{/privacy}","received_events_url":"https://api.github.com/users/umbraco/received_events","type":"Organization","site_admin":false},"repo":{"id":10601208,"name":"Umbraco-CMS","full_name":"umbraco/Umbraco-CMS","owner":{"login":"umbraco","id":1419552,"avatar_url":"https://avatars.githubusercontent.com/u/1419552?v=3","gravatar_id":"","url":"https://api.github.com/users/umbraco","html_url":"https://github.com/umbraco","followers_url":"https://api.github.com/users/umbraco/followers","following_url":"https://api.github.com/users/umbraco/following{/other_user}","gists_url":"https://api.github.com/users/umbraco/gists{/gist_id}","starred_url":"https://api.github.com/users/umbraco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/umbraco/subscriptions","organizations_url":"https://api.github.com/users/umbraco/orgs","repos_url":"https://api.github.com/users/umbraco/repos","events_url":"https://api.github.com/users/umbraco/events{/privacy}","received_events_url":"https://api.github.com/users/umbraco/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/umbraco/Umbraco-CMS","description":"The simple, flexible and friendly ASP.NET CMS used by more than 350.000 websites","fork":false,"url":"https://api.github.com/repos/umbraco/Umbraco-CMS","forks_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/forks","keys_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/keys{/key_id}","collaborators_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/teams","hooks_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/hooks","issue_events_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/events{/number}","events_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/events","assignees_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/assignees{/user}","branches_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/branches{/branch}","tags_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/tags","blobs_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/refs{/sha}","trees_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/trees{/sha}","statuses_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/statuses/{sha}","languages_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/languages","stargazers_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/stargazers","contributors_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/contributors","subscribers_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/subscribers","subscription_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/subscription","commits_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/commits{/sha}","git_commits_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/git/commits{/sha}","comments_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/comments{/number}","issue_comment_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/comments{/number}","contents_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/contents/{+path}","compare_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/compare/{base}...{head}","merges_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/merges","archive_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/downloads","issues_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues{/number}","pulls_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls{/number}","milestones_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/milestones{/number}","notifications_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/labels{/name}","releases_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/releases{/id}","deployments_url":"https://api.github.com/repos/umbraco/Umbraco-CMS/deployments","created_at":"2013-06-10T13:50:46Z","updated_at":"2016-05-17T09:15:53Z","pushed_at":"2016-05-17T11:27:46Z","git_url":"git://github.com/umbraco/Umbraco-CMS.git","ssh_url":"git@github.com:umbraco/Umbraco-CMS.git","clone_url":"https://github.com/umbraco/Umbraco-CMS.git","svn_url":"https://github.com/umbraco/Umbraco-CMS","homepage":"http://umbraco.com","size":190152,"stargazers_count":1208,"watchers_count":1208,"language":"C#","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":973,"mirror_url":null,"open_issues_count":73,"forks":973,"open_issues":73,"watchers":1208,"default_branch":"dev-v7"}},"_links":{"self":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255"},"html":{"href":"https://github.com/umbraco/Umbraco-CMS/pull/1255"},"issue":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/1255"},"comments":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/issues/1255/comments"},"review_comments":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255/comments"},"review_comment":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/pulls/1255/commits"},"statuses":{"href":"https://api.github.com/repos/umbraco/Umbraco-CMS/statuses/512c19a13208ab7091c34174479fad10255b6f4a"}}}}