{"action":"created","comment":{"url":"https://api.github.com/repos/ceph/ceph/pulls/comments/167623326","pull_request_review_id":95457111,"id":167623326,"diff_hunk":"@@ -0,0 +1,1664 @@\n+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-\n+// vim: ts=8 sw=2 smarttab\n+\n+#include \"ReplicatedWriteLog.h\"\n+#include \"include/buffer.h\"\n+#include \"include/Context.h\"\n+#include \"common/deleter.h\"\n+#include \"common/dout.h\"\n+#include \"common/errno.h\"\n+#include \"common/WorkQueue.h\"\n+#include \"librbd/ImageCtx.h\"\n+#include \"librbd/cache/file/ImageStore.h\"\n+#include \"librbd/cache/file/JournalStore.h\"\n+#include \"librbd/cache/file/MetaStore.h\"\n+#include \"librbd/cache/file/StupidPolicy.h\"\n+#include \"librbd/cache/file/Types.h\"\n+#include <map>\n+#include <vector>\n+\n+#define dout_subsys ceph_subsys_rbd\n+#undef dout_prefix\n+#define dout_prefix *_dout << \"librbd::cache::ReplicatedWriteLog: \" << this << \" \" \\\n+                           <<  __func__ << \": \"\n+\n+namespace librbd {\n+namespace cache {\n+\n+using namespace librbd::cache::rwl;\n+using namespace librbd::cache::file;\n+\n+BlockExtent block_extent(uint64_t offset_bytes, uint64_t length_bytes)\n+{\n+  return BlockExtent(offset_bytes / MIN_WRITE_SIZE,\n+\t\t     ((offset_bytes + length_bytes) / MIN_WRITE_SIZE) - 1);\n+}\n+\n+BlockExtent block_extent(ImageCache::Extent& image_extent)\n+{\n+  return block_extent(image_extent.first, image_extent.second);\n+}\n+\n+ImageCache::Extent image_extent(BlockExtent block_extent)\n+{\n+  return ImageCache::Extent(block_extent.block_start * MIN_WRITE_SIZE,\n+\t\t\t    (block_extent.block_end - block_extent.block_start + 1) * MIN_WRITE_SIZE);\n+}\n+\n+namespace rwl {\n+\n+SyncPoint::SyncPoint(CephContext *cct, uint64_t sync_gen_num)\n+  : m_cct(cct), m_sync_gen_num(sync_gen_num),\n+    m_prior_log_entries_persisted_status(0), m_on_sync_point_persisted(NULL) {\n+  m_prior_log_entries_persisted = new C_Gather(cct, NULL);\n+  ldout(cct, 6) << \"sync point\" << m_sync_gen_num << dendl;\n+  /* TODO: Connect m_prior_log_entries_persisted finisher to append this\n+     sync point and on persist complete call on_sync_point_persisted\n+     and delete this object */\n+}\n+SyncPoint::~SyncPoint() {\n+  /* TODO: will m_prior_log_entries_persisted always delete itself? */\n+}\n+\n+WriteLogOperation::WriteLogOperation(WriteLogOperationSet *set, uint64_t image_offset_bytes, uint64_t write_bytes)\n+  : log_entry(new WriteLogEntry(image_offset_bytes, write_bytes)) {\n+  on_write_persist = set->m_extent_ops->new_sub();\n+}\n+\n+WriteLogOperation::~WriteLogOperation() { }\n+\n+/* Called when the write log operation is completed in all log replicas */\n+void WriteLogOperation::complete(int result) {\n+  on_write_persist->complete(result);\n+  delete(this);\n+}\n+\n+WriteLogOperationSet::WriteLogOperationSet(CephContext *cct, SyncPoint *sync_point, bool persist_on_flush,\n+\t\t\t\t\t   BlockExtent extent, Context *on_finish)\n+  : m_cct(cct), m_extent(extent), m_on_finish(on_finish), m_persist_on_flush(persist_on_flush) {\n+  m_on_ops_persist = sync_point->m_prior_log_entries_persisted->new_sub();\n+  m_extent_ops =\n+    new C_Gather(cct,\n+\t\t new FunctionContext( [this](int r) {\n+\t\t     //ldout(m_cct, 6) << \"m_extent_ops completed\" << dendl;\n+\t\t     m_on_ops_persist->complete(r);\n+\t\t     m_on_finish->complete(r);\n+\t\t     delete(this);\n+\t\t   }));\n+}\n+\n+WriteLogOperationSet::~WriteLogOperationSet() { }\n+\n+void WriteLogEntry::add_reader() { reader_count++; }\n+void WriteLogEntry::remove_reader() { reader_count--; }\n+\n+/**\n+ * Add a write log entry to the map. Subsequent queries for blocks\n+ * within this log entry's extent will find this log entry. Portions\n+ * of prior write log entries overlapping with this log entry will\n+ * be replaced in the map by this log entry.\n+ *\n+ * The map_entries field of the log entry object will be updated to\n+ * contain this map entry.\n+ *\n+ * The map_entries fields of all log entries overlapping with this\n+ * entry will be updated to remove the regions that overlap with\n+ * this.\n+ */\n+void WriteLogMap::add_entry(WriteLogEntry *log_entry) {\n+  Mutex::Locker locker(m_lock);\n+  add_entry_locked(log_entry);\n+}\n+\n+void WriteLogMap::add_entries(WriteLogEntries &log_entries) {\n+  Mutex::Locker locker(m_lock);\n+  ldout(m_cct, 20) << dendl;\n+  for (auto &log_entry : log_entries) {\n+    add_entry_locked(log_entry);\n+  }\n+}\n+  \n+/**\n+ * Remove any map entries that refer to the supplied write log\n+ * entry.\n+ */\n+void WriteLogMap::remove_entry(WriteLogEntry *log_entry) {\n+  Mutex::Locker locker(m_lock);\n+  remove_entry_locked(log_entry);\n+}\n+\n+void WriteLogMap::remove_entries(WriteLogEntries &log_entries) {\n+  Mutex::Locker locker(m_lock);\n+  ldout(m_cct, 20) << dendl;\n+  for (auto &log_entry : log_entries) {\n+    remove_entry_locked(log_entry);\n+  }\n+}\n+\n+#if 0\n+/**\n+ * Returns the list of all write log entries that overlap the\n+ * specified block extent.\n+ */\n+WriteLogEntries WriteLogMap::find_entries(BlockExtent block_extent) {\n+  Mutex::Locker locker(m_lock);\n+  ldout(m_cct, 20) << dendl;\n+  return find_entries_locked(block_extent);\n+}\n+#endif\n+\n+/**\n+ * Returns the list of all write log map entries that overlap the\n+ * specified block extent.\n+ */\n+WriteLogMapEntries WriteLogMap::find_map_entries(BlockExtent block_extent) {\n+  Mutex::Locker locker(m_lock);\n+  ldout(m_cct, 20) << dendl;\n+  return find_map_entries_locked(block_extent);\n+}\n+  \n+void WriteLogMap::add_entry_locked(WriteLogEntry *log_entry) {\n+  WriteLogMapEntry map_entry(log_entry);\n+  ldout(m_cct, 20) << \"block_extent=\" << map_entry.block_extent \n+\t\t   << dendl;\n+  assert(m_lock.is_locked_by_me());\n+  WriteLogMapEntries overlap_entries = find_map_entries_locked(map_entry.block_extent);\n+  if (overlap_entries.size()) {\n+    for (auto &entry : overlap_entries) {\n+      ldout(m_cct, 20) << entry << dendl;\n+      if (map_entry.block_extent.block_start <= entry.block_extent.block_start) {\n+\tif (map_entry.block_extent.block_end >= entry.block_extent.block_end) {\n+\t  ldout(m_cct, 20) << \"map entry completely occluded by new log entry\" << dendl;\n+\t  remove_map_entry_locked(entry);\n+\t} else {\n+\t  assert(map_entry.block_extent.block_end < entry.block_extent.block_end);\n+\t  /* The new entry occludes the beginning of the old entry */\n+\t  BlockExtent adjusted_extent(map_entry.block_extent.block_end+1,\n+\t\t\t\t      entry.block_extent.block_end);\n+\t  adjust_map_entry_locked(entry, adjusted_extent);\n+\t}\n+      } else {\n+\tassert(map_entry.block_extent.block_start > entry.block_extent.block_start);\n+\tif (map_entry.block_extent.block_end >= entry.block_extent.block_end) {\n+\t  /* The new entry occludes the end of the old entry */\n+\t  BlockExtent adjusted_extent(entry.block_extent.block_start,\n+\t\t\t\t      map_entry.block_extent.block_start-1);\n+\t  adjust_map_entry_locked(entry, adjusted_extent);\n+\t} else {\n+\t  /* The new entry splits the old entry */\n+\t  split_map_entry_locked(entry, map_entry.block_extent);\n+\t}\n+      }\n+    }\n+  }\n+  add_map_entry_locked(map_entry);\n+}\n+\n+void WriteLogMap::remove_entry_locked(WriteLogEntry *log_entry) {\n+  ldout(m_cct, 20) << \"*log_entry=\" << *log_entry << dendl;\n+  assert(m_lock.is_locked_by_me());\n+  \n+#if 0\n+    assert(cell != nullptr);\n+    auto &detained_block_extent = reinterpret_cast<WriteLogMapExtent &>(\n+      *cell);\n+    ldout(m_cct, 20) << \"block_start=\"\n+                     << detained_block_extent.block_extent.block_start << \", \"\n+                     << \"block_end=\"\n+                     << detained_block_extent.block_extent.block_end << \", \"\n+                     << \"pending_ops=\"\n+                     << (detained_block_extent.block_operations.empty() ?\n+                          0 : detained_block_extent.block_operations.size() - 1)\n+                     << dendl;\n+\n+    *block_operations = std::move(detained_block_extent.block_operations);\n+    m_detained_block_extents.erase(detained_block_extent.block_extent);\n+    m_free_detained_block_extents.push_back(detained_block_extent);\n+#endif\n+}\n+\n+void WriteLogMap::add_map_entry_locked(WriteLogMapEntry &map_entry)\n+{\n+  m_block_to_log_entry_map.insert(map_entry);\n+  map_entry.log_entry->referring_map_entries++;\n+}\n+\n+void WriteLogMap::remove_map_entry_locked(WriteLogMapEntry &map_entry)\n+{\n+  auto it = m_block_to_log_entry_map.find(map_entry);\n+  assert(it != m_block_to_log_entry_map.end());\n+\n+  WriteLogMapEntry erased = *it;\n+  m_block_to_log_entry_map.erase(it);\n+  erased.log_entry->referring_map_entries--;\n+  if (0 == erased.log_entry->referring_map_entries) {\n+    ldout(m_cct, 20) << \"log entry has zero map entries: \" << erased.log_entry << dendl;\n+  }\n+}\n+\n+void WriteLogMap::adjust_map_entry_locked(WriteLogMapEntry &map_entry, BlockExtent &new_extent)\n+{\n+  auto it = m_block_to_log_entry_map.find(map_entry);\n+  assert(it != m_block_to_log_entry_map.end());\n+\n+  WriteLogMapEntry adjusted = *it;\n+  m_block_to_log_entry_map.erase(it);\n+\n+  m_block_to_log_entry_map.insert(WriteLogMapEntry(new_extent, adjusted.log_entry));\n+}\n+\n+void WriteLogMap::split_map_entry_locked(WriteLogMapEntry &map_entry, BlockExtent &removed_extent)\n+{\n+  auto it = m_block_to_log_entry_map.find(map_entry);\n+  assert(it != m_block_to_log_entry_map.end());\n+\n+  WriteLogMapEntry split = *it;\n+  m_block_to_log_entry_map.erase(it);\n+\n+  BlockExtent left_extent(split.block_extent.block_start,\n+\t\t\t  removed_extent.block_start-1);\n+  m_block_to_log_entry_map.insert(WriteLogMapEntry(left_extent, split.log_entry));\n+\n+  BlockExtent right_extent(removed_extent.block_end+1,\n+\t\t\t   split.block_extent.block_end);\n+  m_block_to_log_entry_map.insert(WriteLogMapEntry(right_extent, split.log_entry));\n+\n+  split.log_entry->referring_map_entries++;\n+}\n+\n+#if 0\n+WriteLogEntries WriteLogMap::find_entries_locked(BlockExtent block_extent) {\n+  WriteLogEntries overlaps;\n+  ldout(m_cct, 20) << \"block_extent=\" << block_extent << dendl;\n+  \n+  assert(m_lock.is_locked_by_me());\n+  WriteLogMapEntries map_entries = find_map_entries_locked(block_extent);\n+  for (auto &map_entry : map_entries) {\n+    overlaps.emplace_back(map_entry.log_entry);\n+  }\n+  return overlaps;\n+}\n+#endif\n+\n+/**\n+ * TODO: Generalize this to do some arbitrary thing to each map\n+ * extent, instead of returning a list.\n+ */\n+WriteLogMapEntries WriteLogMap::find_map_entries_locked(BlockExtent &block_extent) {\n+  WriteLogMapEntries overlaps;\n+  \n+  ldout(m_cct, 20) << \"block_extent=\" << block_extent << dendl;\n+  assert(m_lock.is_locked_by_me());\n+  auto p = m_block_to_log_entry_map.equal_range(WriteLogMapEntry(block_extent));\n+  ldout(m_cct, 20) << \"count=\" << std::distance(p.first, p.second) << dendl;\n+  for ( auto i = p.first; i != p.second; ++i ) {\n+    WriteLogMapEntry entry = *i;\n+    overlaps.emplace_back(entry);\n+    ldout(m_cct, 20) << entry << dendl;\n+  }\n+  return overlaps;\n+}\n+\n+bool is_block_aligned(const ImageCache::Extents &image_extents) {\n+  for (auto &extent : image_extents) {\n+    if (extent.first % MIN_WRITE_SIZE != 0 || extent.second % MIN_WRITE_SIZE != 0) {\n+      return false;\n+    }\n+  }\n+  return true;\n+}\n+\n+/**\n+ * A chain of requests that stops on the first failure\n+ */\n+struct C_BlockIORequest : public Context {\n+  CephContext *m_cct;\n+  C_BlockIORequest *next_block_request;\n+\n+  C_BlockIORequest(CephContext *cct, C_BlockIORequest *next_block_request)\n+    : m_cct(cct), next_block_request(next_block_request) {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+  ~C_BlockIORequest() {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+\n+  virtual void finish(int r) override {\n+    ldout(m_cct, 20) << \"(\" << get_name() << \"): r=\" << r << dendl;\n+\n+    if (NULL != next_block_request) {\n+      if (r < 0) {\n+\t// abort the chain of requests upon failure\n+\tnext_block_request->complete(r);\n+      } else {\n+\t// execute next request in chain\n+\tnext_block_request->send();\n+      }\n+    }\n+  }\n+\n+  virtual void send() = 0;\n+  virtual const char *get_name() const = 0;\n+};\n+\n+/**\n+ * A request that can be deferred in a BlockGuard to sequence\n+ * overlapping operatoins.\n+ */\n+struct C_GuardedBlockIORequest : public C_BlockIORequest {\n+private:\n+  CephContext *m_cct;\n+  BlockGuardCell* m_cell;\n+public:\n+  C_GuardedBlockIORequest(CephContext *cct, C_BlockIORequest *next_block_request)\n+    : C_BlockIORequest(cct, next_block_request), m_cct(cct), m_cell(NULL) {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+  C_GuardedBlockIORequest(CephContext *cct)\n+    : C_BlockIORequest(cct, NULL), m_cct(cct), m_cell(NULL) {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+  ~C_GuardedBlockIORequest() {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+  C_GuardedBlockIORequest(const C_GuardedBlockIORequest&) = delete;\n+  C_GuardedBlockIORequest &operator=(const C_GuardedBlockIORequest&) = delete;\n+\n+  virtual void send() = 0;\n+  virtual const char *get_name() const = 0;\n+  void set_cell(BlockGuardCell *cell) {\n+    ldout(m_cct, 20) << this << dendl;\n+    assert(NULL != cell);\n+    m_cell = cell;\n+  }\n+  BlockGuardCell *get_cell(void) {\n+    ldout(m_cct, 20) << this << dendl;\n+    return m_cell;\n+  }\n+};\n+\n+} // namespace rwl\n+\n+template <typename I>\n+ReplicatedWriteLog<I>::ReplicatedWriteLog(ImageCtx &image_ctx)\n+  : m_image_ctx(image_ctx), \n+    m_log_pool(NULL), m_log_pool_size(DEFAULT_POOL_SIZE),\n+    m_total_log_entries(0), m_free_log_entries(0), \n+    m_image_writeback(image_ctx), m_image_cache(image_ctx),\n+    m_write_log_guard(image_ctx.cct),\n+    m_first_free_entry(0), m_first_valid_entry(0),\n+    m_current_sync_gen(0), m_current_sync_point(NULL),\n+    m_last_op_sequence_num(0),\n+    m_persist_on_write_until_flush(true), m_persist_on_flush(false),\n+    m_flush_seen(false),\n+    m_log_append_lock(\"librbd::cache::ReplicatedWriteLog::m_log_append_lock\"),\n+    m_lock(\"librbd::cache::ReplicatedWriteLog::m_lock\"),\n+    m_persist_finisher(image_ctx.cct),\n+    m_log_append_finisher(image_ctx.cct),\n+    m_on_persist_finisher(image_ctx.cct),\n+    m_blocks_to_log_entries(image_ctx.cct) {\n+  m_persist_finisher.start();\n+  m_log_append_finisher.start();\n+  m_on_persist_finisher.start();\n+}\n+\n+template <typename I>\n+ReplicatedWriteLog<I>::~ReplicatedWriteLog() {\n+  m_persist_finisher.wait_for_empty();\n+  m_log_append_finisher.wait_for_empty();\n+  m_on_persist_finisher.wait_for_empty();\n+}\n+\n+template <typename ExtentsType>\n+class ExtentsSummary {\n+public:\n+  uint64_t total_bytes;\n+  uint64_t first_image_byte;\n+  uint64_t last_image_byte;\n+  uint64_t first_block;\n+  uint64_t last_block;\n+  ExtentsSummary(const ExtentsType &extents) {\n+    total_bytes = 0;\n+    first_image_byte = 0;\n+    last_image_byte = 0;\n+    first_block = 0;\n+    last_block = 0;\n+    if (extents.empty()) return;\n+    first_image_byte = extents.front().first;\n+    last_image_byte = first_image_byte + extents.front().second;\n+    for (auto &extent : extents) {\n+      total_bytes += extent.second;\n+      if (extent.first < first_image_byte) {\n+\tfirst_image_byte = extent.first;\n+      }\n+      if ((extent.first + extent.second) > last_image_byte) {\n+\tlast_image_byte = extent.first + extent.second;\n+      }\n+    }\n+    first_block = first_image_byte / MIN_WRITE_SIZE;\n+    last_block = last_image_byte / MIN_WRITE_SIZE;\n+  }\n+};\n+\n+struct ImageExtentBuf : public ImageCache::Extent {\n+public:\n+  buffer::raw *m_buf;\n+  ImageExtentBuf(ImageCache::Extent extent, buffer::raw *buf)\n+    : ImageCache::Extent(extent), m_buf(buf) {}\n+  ImageExtentBuf(ImageCache::Extent extent)\n+    : ImageExtentBuf(extent, NULL) {}\n+};\n+typedef std::vector<ImageExtentBuf> ImageExtentBufs;\n+\n+struct C_ReadRequest : public Context {\n+  CephContext *m_cct;\n+  Context *m_on_finish;\n+  ImageCache::Extents m_miss_extents; // move back to caller\n+  ImageExtentBufs m_read_extents;\n+  bufferlist m_miss_bl;\n+  bufferlist *m_out_bl;\n+  \n+  C_ReadRequest(CephContext *cct, bufferlist *out_bl, Context *on_finish)\n+    : m_cct(cct), m_on_finish(on_finish), m_out_bl(out_bl) {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+  ~C_ReadRequest() {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+\n+  virtual void finish(int r) override {\n+    ldout(m_cct, 20) << \"(\" << get_name() << \"): r=\" << r << dendl;\n+    if (r >= 0) {\n+      /*\n+       * At this point the miss read has completed. We'll iterate through\n+       * m_read_extents and produce *m_out_bl by assembling pieces of m_miss_bl\n+       * and the individual hit extent bufs in the read extents that represent\n+       * hits.\n+       */\n+      uint64_t miss_bl_offset = 0;\n+      for (auto &extent : m_read_extents) {\n+\tif (NULL == extent.m_buf) {\n+\t  /* This was a miss. */\n+\t  bufferlist miss_extent_bl;\n+\t  miss_extent_bl.substr_of(m_miss_bl, miss_bl_offset, extent.second);\n+\t  /* Add this read miss bullerlist to the output bufferlist */\n+\t  m_out_bl->claim_append(miss_extent_bl);\n+\t  /* Consume these bytes in the read miss bufferlist */\n+\t  miss_bl_offset += extent.second;\n+\t} else {\n+\t  /* This was a hit */\n+\t  bufferlist hit_extent_bl;\n+\t  hit_extent_bl.append(extent.m_buf);\n+\t  m_out_bl->claim_append(hit_extent_bl);\n+\t}\n+      }\n+    }\n+    ldout(m_cct, 20) << \"(\" << get_name() << \"): r=\" << r << \" bl=\" << *m_out_bl << dendl;\n+    m_on_finish->complete(r);\n+  }\n+  \n+  virtual const char *get_name() const {\n+    return \"C_ReadRequest\";\n+  }\n+}; \n+ \n+template <typename I>\n+void ReplicatedWriteLog<I>::aio_read(Extents &&image_extents, bufferlist *bl,\n+                                 int fadvise_flags, Context *on_finish) {\n+  CephContext *cct = m_image_ctx.cct;\n+  C_ReadRequest *read_ctx = new C_ReadRequest(cct, bl, on_finish);\n+  ldout(cct, 5) << \"image_extents=\" << image_extents << \", \"\n+\t\t<< \"bl=\" << bl << \", \"\n+\t\t<< \"on_finish=\" << on_finish << dendl;\n+\n+  bl->clear();\n+\n+  // TODO: Handle unaligned IO.\n+  if (!is_block_aligned(image_extents)) {\n+    lderr(cct) << \"unaligned read fails\" << dendl;\n+    for (auto &extent : image_extents) {\n+      lderr(cct) << \"start: \" << extent.first << \" length: \" << extent.second << dendl;\n+    }\n+    on_finish->complete(-EINVAL);\n+    return;\n+  }\n+\n+  // TODO handle fadvise flags\n+\n+  /*\n+   * The strategy here is to look up all the WriteLogMapEntries that overlap\n+   * this read, and iterate through those to separate this read into hits and\n+   * misses. A new Extents object is produced here with Extents for each miss\n+   * region. The miss Extents is then passed on to the read cache below RWL. We\n+   * also produce an ImageExtentBufs for all the extents (hit or miss) in this\n+   * read. When the read from the lower cache layer completes, we iterate\n+   * through the ImageExtentBufs and insert buffers for each cache hit at the\n+   * appropriate spot in the buferlist returned from below for the miss\n+   * read. The buffers we insert here refer directly to regions of various\n+   * write log entry data buffers.\n+   *\n+   * TBD: These buffer objects hold a reference on those write log entries to\n+   * prevent them from being retired from the log while the read is\n+   * completing. The WriteLogEntry references are released by the buffer\n+   * destructor.\n+   */\n+  for (auto &extent : image_extents) {\n+    uint64_t extent_offset = 0;\n+    WriteLogMapEntries map_entries = m_blocks_to_log_entries.find_map_entries(block_extent(extent));\n+    for (auto &entry : map_entries) {\n+      ImageCache::Extent entry_image_extent(image_extent(entry.block_extent));\n+      /* If this map entry starts after the current image extent offset ... */\n+      if (entry_image_extent.first > extent.first + extent_offset) {\n+\t/* ... add range before map_entry to miss extents */\n+\tuint64_t miss_extent_start = extent.first + extent_offset;\n+\tuint64_t miss_extent_length = entry_image_extent.first - miss_extent_start;\n+\tImageCache::Extent miss_extent(miss_extent_start, miss_extent_length);\n+\tread_ctx->m_miss_extents.push_back(miss_extent);\n+\t/* Add miss range to read extents */\n+\tImageExtentBuf miss_extent_buf(miss_extent);\n+\tread_ctx->m_read_extents.push_back(miss_extent_buf);\n+\textent_offset += miss_extent_length;\n+      }\n+      assert(entry_image_extent.first <= extent.first + extent_offset);\n+      uint64_t entry_offset = 0;\n+      /* If this map entry starts before the current image extent offset ... */\n+      if (entry_image_extent.first < extent.first + extent_offset) {\n+\t/* ... compute offset into log entry for this read extent */\n+\tentry_offset = (extent.first + extent_offset) - entry_image_extent.first;\n+      }\n+      /* This read hit ends at the end of the extent or the end of the log\n+\t entry, whichever is less. */\n+      uint64_t entry_hit_length = min(entry_image_extent.second - entry_offset,\n+\t\t\t\t      extent.second - extent_offset);\n+      ImageCache::Extent hit_extent(entry_image_extent.first, entry_hit_length);\n+      /* Offset of the map entry into the log entry's buffer */\n+      uint64_t map_entry_buffer_offset = entry_image_extent.first - entry.log_entry->ram_entry.image_offset_bytes;\n+      /* Offset into the log entry buffer of this read hit */\n+      uint64_t read_buffer_offset = map_entry_buffer_offset + entry_offset;\n+      /* Create buffer object referring to pmem pool for this read hit */\n+      WriteLogEntry *log_entry = entry.log_entry;\n+      ldout(cct, 5) << \"adding reader: log_entry=\" << *log_entry << dendl;\n+      log_entry->add_reader();\n+      buffer::raw *hit_buf =\n+\tbuffer::claim_buffer(entry_hit_length,\n+\t\t\t     (char*)(log_entry->pmem_buffer + read_buffer_offset),\n+\t\t\t     make_deleter([this, log_entry]\n+\t\t\t\t\t  {\n+\t\t\t\t\t    CephContext *cct = m_image_ctx.cct;\n+\t\t\t\t\t    ldout(cct, 5) << \"removing reader: log_entry=\"\n+\t\t\t\t\t\t\t  << *log_entry << dendl;\n+\t\t\t\t\t    log_entry->remove_reader();\n+\t\t\t\t\t  }));\n+      /* Add hit extent to read extents */\n+      ImageExtentBuf hit_extent_buf(hit_extent, hit_buf);\n+      read_ctx->m_read_extents.push_back(hit_extent_buf);\n+      /* Exclude RWL hit range from buffer and extent */\n+      extent_offset += entry_hit_length;\n+      ldout(cct, 20) << entry << dendl;\n+    }\n+    /* If the last map entry didn't consume the entire image extent ... */\n+    if (extent.second > extent_offset) {\n+      /* ... add the rest of this extent to miss extents */\n+      uint64_t miss_extent_start = extent.first + extent_offset;\n+      uint64_t miss_extent_length = extent.second - extent_offset;\n+      ImageCache::Extent miss_extent(miss_extent_start, miss_extent_length);\n+      read_ctx->m_miss_extents.push_back(miss_extent);\n+      /* Add miss range to read extents */\n+      ImageExtentBuf miss_extent_buf(miss_extent);\n+      read_ctx->m_read_extents.push_back(miss_extent_buf);\n+      extent_offset += miss_extent_length;\n+    }\n+  }\n+  \n+  ldout(cct, 5) << \"miss_extents=\" << read_ctx->m_miss_extents << \", \"\n+\t\t<< \"miss_bl=\" << read_ctx->m_miss_bl << dendl;\n+\n+  if (read_ctx->m_miss_extents.empty()) {\n+    /* All of this read comes from RWL */\n+    read_ctx->complete(0);\n+  } else {\n+    /* Pass the read misses on to the layer below RWL */\n+    if (m_image_ctx.persistent_cache_enabled) {\n+      m_image_cache.aio_read(std::move(read_ctx->m_miss_extents), &read_ctx->m_miss_bl, fadvise_flags, read_ctx);\n+    } else {\n+      m_image_writeback.aio_read(std::move(read_ctx->m_miss_extents), &read_ctx->m_miss_bl, fadvise_flags, read_ctx);\n+    }\n+  }\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::detain_guarded_request(GuardedRequest &&req)\n+{\n+  CephContext *cct = m_image_ctx.cct;\n+  ldout(cct, 20) << dendl;\n+  \n+  BlockGuardCell *cell;\n+  int r = m_write_log_guard.detain({req.first_block_num, req.last_block_num},\n+\t\t\t\t   &req, &cell);\n+  if (r < 0) {\n+    lderr(cct) << \"failed to detain guarded request: \" << cpp_strerror(r)\n+\t\t   << dendl;\n+    m_image_ctx.op_work_queue->queue(req.on_guard_acquire, r);\n+    return;\n+  } else if (r > 0) { \n+    ldout(cct, 6) << \"detaining guarded request due to in-flight requests: \"\n+                   << \"start=\" << req.first_block_num << \", \"\n+\t\t   << \"end=\" << req.last_block_num << dendl;\n+    return;\n+  }\n+\n+  ldout(cct, 6) << \"in-flight request cell: \" << cell << dendl;\n+  req.on_guard_acquire->acquired(cell);\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::release_guarded_request(BlockGuardCell *cell)\n+{\n+  CephContext *cct = m_image_ctx.cct;\n+  ldout(cct, 6) << \"cell=\" << cell << dendl;\n+\n+  WriteLogGuard::BlockOperations block_ops;\n+  m_write_log_guard.release(cell, &block_ops);\n+\n+  for (auto &op : block_ops) {\n+    detain_guarded_request(std::move(op));\n+  }\n+}\n+\n+/**\n+ * This is the custodian of the BlockGuard cell for this write, and\n+ * the state information about the progress of this write. This object\n+ * lives until the write is persisted in all (live) log replicas.\n+ * User write op may be completed from here before the write\n+ * persists. Block guard is not released until the write persists\n+ * everywhere (this is how we guarantee to each log repica that they\n+ * will never see overlapping writes).\n+ */\n+struct C_WriteRequest : public C_GuardedBlockIORequest {\n+  CephContext *m_cct;\n+  ImageCache::Extents image_extents;\n+  bufferlist bl;\n+  int fadvise_flags;\n+  Context *user_req; /* User write request */\n+  Context *_on_finish; /* Block guard release */\n+  std::atomic<bool> m_user_req_completed;\n+  C_WriteRequest(CephContext *cct, ImageCache::Extents &&image_extents,\n+\t\t bufferlist&& bl, int fadvise_flags, Context *user_req)\n+    : C_GuardedBlockIORequest(cct), m_cct(cct), image_extents(std::move(image_extents)),\n+      bl(std::move(bl)), fadvise_flags(fadvise_flags),\n+      user_req(user_req), _on_finish(NULL), m_user_req_completed(false) {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+\n+  ~C_WriteRequest() {\n+    ldout(m_cct, 99) << this << dendl;\n+  }\n+\n+  void complete_user_request(int r) {\n+    bool initial = false;\n+    if (m_user_req_completed.compare_exchange_strong(initial, true)) {\n+      ldout(m_cct, 15) << this << \" completing user req\" << dendl;\n+      user_req->complete(r);\n+    } else {\n+      ldout(m_cct, 20) << this << \" user req already completed\" << dendl;\n+    }\n+  }\n+\n+  virtual void send() override {\n+    /* Should never be called */\n+    ldout(m_cct, 6) << this << \" unexpected\" << dendl;\n+  }\n+  \n+  virtual void finish(int r) {\n+    ldout(m_cct, 6) << this << dendl;\n+\n+    complete_user_request(r);\n+    _on_finish->complete(0);\n+  }\n+\n+  virtual const char *get_name() const override {\n+    return \"C_WriteRequest\";\n+  }  \n+};\n+\n+/*\n+ * Performs the log event append operation for all of the scheduled\n+ * events.\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::append_scheduled_ops(void)\n+{\n+  WriteLogOperations ops;\n+  int append_result = 0;\n+\n+  {\n+    Mutex::Locker locker(m_lock);\n+    if (m_ops_to_append.empty()) {\n+      return;\n+    }\n+  }\n+  {\n+    Mutex::Locker locker(m_log_append_lock);\n+\n+    {\n+      Mutex::Locker locker(m_lock);\n+      std::swap(ops, m_ops_to_append);\n+    }\n+    \n+    if (ops.size()) {\n+      alloc_op_log_entries(ops);\n+      append_result = append_op_log_entries(ops);\n+    }\n+  }\n+\n+  if (ops.size()) {\n+    complete_op_log_entries(ops, append_result);\n+  }\n+}\n+\n+/*\n+ * Takes custody of ops. They'll all get their log entries appended,\n+ * and have their on_write_persist contexts completed once they and\n+ * all prior log entries are persisted everywhere.\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::schedule_append(WriteLogOperations &ops)\n+{\n+  {\n+    Mutex::Locker locker(m_lock);\n+\n+    m_ops_to_append.splice(m_ops_to_append.end(), ops);\n+  }\n+\n+  m_log_append_finisher.queue(new FunctionContext([this](int r) {\n+\tappend_scheduled_ops();\n+      }));\n+}\n+\n+/*\n+ * Performs the pmem buffer flush on all scheduled ops, then schedules\n+ * the log event append operation for all of them.\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::flush_then_append_scheduled_ops(void)\n+{\n+  WriteLogOperations ops;\n+  {\n+    Mutex::Locker locker(m_lock);\n+    std::swap(ops, m_ops_to_flush);\n+  }\n+\n+  /* Ops subsequently scheduled for flush may finish before these,\n+   * which is fine. We're unconcerned with completion order until we\n+   * get to the log message append step. */\n+  if (ops.size()) {\n+    flush_pmem_buffer(ops);\n+    schedule_append(ops);\n+  }\n+}\n+\n+/*\n+ * Takes custody of ops. They'll all get their pmem blocks flushed,\n+ * then get their log entries appended.\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::schedule_flush_and_append(WriteLogOperations &ops)\n+{\n+  {\n+    Mutex::Locker locker(m_lock);\n+\n+    m_ops_to_flush.splice(m_ops_to_flush.end(), ops);\n+  }\n+\n+  m_persist_finisher.queue(new FunctionContext([this](int r) {\n+\tflush_then_append_scheduled_ops();\n+      }));\n+}\n+\n+/*\n+ * Flush the pmem regions for the data blocks of a set of operations\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::flush_pmem_buffer(WriteLogOperations &ops)\n+{\n+  for (auto &operation : ops) {\n+    pmemobj_flush(m_log_pool, operation->log_entry->pmem_buffer, operation->log_entry->ram_entry.write_bytes);\n+  }\n+  /* Drain once for all */\n+  pmemobj_drain(m_log_pool);\n+}\n+\n+/*\n+ * Allocate the (already resrved) write log entries for a set of operations.\n+ *\n+ * Locking:\n+ * Acquires m_lock\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::alloc_op_log_entries(WriteLogOperations &ops)\n+{\n+  TOID(struct WriteLogPoolRoot) pool_root;\n+  pool_root = POBJ_ROOT(m_log_pool, struct WriteLogPoolRoot);\n+  struct WriteLogPmemEntry *pmem_log_entries = D_RW(D_RW(pool_root)->log_entries);\n+\n+  /* Allocate the (already reserved) log entries */\n+  {\n+    Mutex::Locker locker(m_lock);\n+\n+    for (auto &operation : ops) {\n+      operation->log_entry->log_entry_index = m_first_free_entry;\n+      m_first_free_entry = (m_first_free_entry + 1) % m_total_log_entries;\n+      operation->log_entry->pmem_entry = &pmem_log_entries[operation->log_entry->log_entry_index];\n+      operation->log_entry->ram_entry.entry_valid = 1;\n+    }\n+  }\n+}\n+\n+/*\n+ * Flush the persistent write log entries set of ops. The entries must\n+ * be contiguous in persistemt memory.\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::flush_op_log_entries(WriteLogOperations &ops)\n+{\n+  if (ops.empty()) return;\n+\n+  if (ops.size() > 1) {\n+    assert(ops.front()->log_entry->pmem_entry < ops.back()->log_entry->pmem_entry);\n+  }\n+  \n+  pmemobj_flush(m_log_pool,\n+\t\tops.front()->log_entry->pmem_entry,\n+\t\tops.size() * sizeof(*(ops.front()->log_entry->pmem_entry)));\n+}\n+\n+/*\n+ * Write and persist the (already allocated) write log entries and\n+ * data buffer allocations for a set of ops. The data buffer for each\n+ * of these must already have been persisted to its reserved area.\n+ */\n+template <typename I>\n+int ReplicatedWriteLog<I>::append_op_log_entries(WriteLogOperations &ops)\n+{\n+  CephContext *cct = m_image_ctx.cct;\n+  WriteLogOperations entries_to_flush;\n+  TOID(struct WriteLogPoolRoot) pool_root;\n+  pool_root = POBJ_ROOT(m_log_pool, struct WriteLogPoolRoot);\n+  \n+  if (ops.empty()) return 0;\n+\n+  /* Write log entries to ring and persist */\n+  for (auto &operation : ops) {\n+    if (!entries_to_flush.empty()) {\n+      /* Flush these and reset the list if the current entry wraps to the \n+       * tail of the ring */\n+      if (entries_to_flush.back()->log_entry->log_entry_index >\n+\t  operation->log_entry->log_entry_index) {\n+\tflush_op_log_entries(entries_to_flush);\n+\tentries_to_flush.clear();\n+      }\n+    }\n+    *operation->log_entry->pmem_entry = operation->log_entry->ram_entry;\n+    entries_to_flush.push_back(operation);\n+  }\n+  flush_op_log_entries(entries_to_flush);\n+  \n+  /* Drain once for all */\n+  pmemobj_drain(m_log_pool);\n+\n+  /* \n+   * Atomically advance the log head pointer and publish the\n+   * allocations for all the data buffers they refer to.\n+   */\n+  TX_BEGIN(m_log_pool) {\n+    D_RW(pool_root)->first_free_entry = m_first_free_entry;\n+    for (auto &operation : ops) {\n+      pmemobj_tx_publish(&operation->buffer_alloc_action, 1);\n+    }  \n+  } TX_ONCOMMIT {\n+  } TX_ONABORT {\n+    lderr(cct) << \"failed to commit log entries (\" << m_log_pool_name << \")\" << dendl;\n+    return(-EIO);\n+  } TX_FINALLY {\n+  } TX_END;\n+  \n+  return 0;\n+}\n+\n+/*\n+ * Complete a set of write ops with the result of append_op_entries.\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::complete_op_log_entries(WriteLogOperations &ops, int result)\n+{\n+  for (auto &operation : ops) {\n+    WriteLogOperation *op = operation;\n+    m_on_persist_finisher.queue(new FunctionContext([this, op, result](int r) {\n+\t  op->complete(result);\n+\t}));\n+  }\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::complete_write_req(C_WriteRequest *write_req, int result)\n+{\n+  CephContext *cct = m_image_ctx.cct;\n+  ldout(cct, 6) << \"write_req=\" << write_req << dendl;\n+  ldout(cct, 6) << \"write_req=\" << write_req << \" cell=\" << write_req->get_cell() << dendl;\n+  assert(NULL != write_req->get_cell());\n+  release_guarded_request(write_req->get_cell());\n+}\n+\n+/**\n+ * Takes custody of write_req.\n+ *\n+ * Locking:\n+ * Acquires m_lock\n+ */\n+template <typename I>\n+void ReplicatedWriteLog<I>::dispatch_aio_write(C_WriteRequest *write_req)\n+{\n+  CephContext *cct = m_image_ctx.cct;\n+  \n+  TOID(struct WriteLogPoolRoot) pool_root;\n+  pool_root = POBJ_ROOT(m_log_pool, struct WriteLogPoolRoot);\n+\n+  ldout(cct, 6) << \"write_req=\" << write_req << \" cell=\" << write_req->get_cell() << dendl;\n+  ldout(cct,6) << \"bl=[\" << write_req->bl << \"]\" << dendl;\n+\n+  ExtentsSummary<ImageCache::Extents> image_extents_summary(write_req->image_extents);\n+\n+  WriteLogEntries log_entries;\n+  WriteLogOperationSet *set =\n+    new WriteLogOperationSet(cct, m_current_sync_point, m_persist_on_flush,\n+\t\t\t     BlockExtent(image_extents_summary.first_block,\n+\t\t\t\t\t image_extents_summary.last_block),\n+\t\t\t     new C_OnFinisher(write_req, &m_on_persist_finisher));\n+  {\n+    uint64_t buffer_offset = 0;\n+    Mutex::Locker locker(m_lock);\n+    /* TODO: If there isn't space for this whole write, defer it */\n+    if (m_free_log_entries < write_req->image_extents.size()) {\n+      ldout(cct, 6) << \"wait for \" << write_req->image_extents.size() << \" log entries\" << dendl;\n+    } else {\n+      for (auto &extent : write_req->image_extents) {\n+\t/* operation->on_write_persist connected to m_prior_log_entries_persisted Gather */\n+\tWriteLogOperation *operation =\n+\t  new WriteLogOperation(set, extent.first, extent.second);\n+\tset->operations.emplace_back(operation);\n+\tlog_entries.emplace_back(operation->log_entry);\n+\n+\t/* Reserve both log entry and data block space */\n+\tm_free_log_entries--;\n+\n+\t/* Allocate data blocks */\n+\toperation->log_entry->ram_entry.has_data = 1;\n+\t/* TODO: Handle failure */\n+\tunsigned int alloc_bytes = MIN_WRITE_ALLOC_SIZE;\n+\tif (operation->log_entry->ram_entry.write_bytes > alloc_bytes) {\n+\t  alloc_bytes = operation->log_entry->ram_entry.write_bytes;\n+\t}\n+\toperation->log_entry->ram_entry.write_data = pmemobj_reserve(m_log_pool,\n+\t\t\t\t\t\t\t\t     &operation->buffer_alloc_action,\n+\t\t\t\t\t\t\t\t     alloc_bytes,\n+\t\t\t\t\t\t\t\t     0 /* Object type */);\n+\tassert(!TOID_IS_NULL(operation->log_entry->ram_entry.write_data));\n+\toperation->log_entry->pmem_buffer = D_RW(operation->log_entry->ram_entry.write_data);\n+\toperation->log_entry->ram_entry.sync_gen_number = m_current_sync_gen;\n+\tif (set->m_persist_on_flush) {\n+\t  /* Persist on flush. Sequence #0 is never used. */\n+\t  operation->log_entry->ram_entry.write_sequence_number = 0;\n+\t} else {\n+\t  /* Persist on write */\n+\t  operation->log_entry->ram_entry.write_sequence_number = ++m_last_op_sequence_num;\n+\t  operation->log_entry->ram_entry.sequenced = 1;\n+\t}\n+\toperation->log_entry->ram_entry.sync_point = 0;\n+\toperation->log_entry->ram_entry.unmap = 0;\n+\toperation->bl.substr_of(write_req->bl, buffer_offset,\n+\t\t\t\toperation->log_entry->ram_entry.write_bytes);\n+\tbuffer_offset += operation->log_entry->ram_entry.write_bytes;\n+\tldout(cct, 6) << \"operation=[\" << *operation << \"]\" << dendl;\n+      }\n+    }\n+  }\n+\n+  m_async_op_tracker.start_op();\n+  write_req->_on_finish =\n+    new FunctionContext([this, write_req](int r) {\n+\tcomplete_write_req(write_req, r);\n+\tm_async_op_tracker.finish_op();\n+      });\n+\n+  /* All extent ops subs created */\n+  set->m_extent_ops->activate();\n+  \n+  /* Write data */\n+  for (auto &operation : set->operations) {\n+    bufferlist::iterator i(&operation->bl);\n+    ldout(cct, 6) << operation->bl << dendl;\n+    i.copy((unsigned)operation->log_entry->ram_entry.write_bytes, (char*)operation->log_entry->pmem_buffer);\n+  }\n+\n+  m_blocks_to_log_entries.add_entries(log_entries);\n+  \n+  if (set->m_persist_on_flush) {\n+    /* \n+     * We're done with the caller's buffer, and not guaranteeing\n+     * persistence until the next flush. The block guard for this\n+     * write_req will not be released until the write is persisted\n+     * everywhere.\n+     */\n+    write_req->complete_user_request(0);\n+  }\n+  \n+  WriteLogOperations ops_copy = set->operations;\n+  schedule_flush_and_append(ops_copy);\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::aio_write(Extents &&image_extents,\n+                                  bufferlist&& bl,\n+                                  int fadvise_flags,\n+                                  Context *on_finish) {\n+  CephContext *cct = m_image_ctx.cct;\n+  //ldout(cct, 20) << \"image_extents=\" << image_extents << \", \"\n+  //               << \"on_finish=\" << on_finish << dendl;\n+\n+  {\n+    RWLock::RLocker snap_locker(m_image_ctx.snap_lock);\n+    if (m_image_ctx.snap_id != CEPH_NOSNAP || m_image_ctx.read_only) {\n+      on_finish->complete(-EROFS);\n+      return;\n+    }\n+  }\n+\n+  // TODO: Handle unaligned IO. RMW must block overlapping writes (to\n+  // the limits of the read phase) and wait for prior overlaping\n+  // writes to complete.\n+  if (!is_block_aligned(image_extents)) {\n+    lderr(cct) << \"unaligned write fails\" << dendl;\n+    for (auto &extent : image_extents) {\n+      lderr(cct) << \"start: \" << extent.first << \" length: \" << extent.second << dendl;\n+    }\n+    on_finish->complete(-EINVAL);\n+    return;\n+  }\n+\n+  ExtentsSummary<ImageCache::Extents> image_extents_summary(image_extents);\n+\n+  C_WriteRequest *write_req =\n+    new C_WriteRequest(cct, std::move(image_extents), std::move(bl), fadvise_flags, on_finish);\n+\n+  /* The lambda below will be called when the block guard for all\n+   * blocks affected by this write is obtained */\n+  GuardedRequestFunctionContext *guarded_ctx =\n+    new GuardedRequestFunctionContext([this, write_req](BlockGuardCell *cell) {\n+      CephContext *cct = m_image_ctx.cct;\n+      ldout(cct, 6) << \"write_req=\" << write_req << \" cell=\" << cell << dendl;\n+\n+      assert(NULL != cell);\n+      write_req->set_cell(cell);\n+\n+      /* TODO: Defer write_req until log resources available */\n+      dispatch_aio_write(write_req);\n+    });\n+\n+  GuardedRequest guarded(image_extents_summary.first_block,\n+\t\t\t image_extents_summary.last_block,\n+\t\t\t guarded_ctx);\n+  detain_guarded_request(std::move(guarded));\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::aio_discard(uint64_t offset, uint64_t length,\n+                                    bool skip_partial_discard, Context *on_finish) {\n+  CephContext *cct = m_image_ctx.cct;\n+  ldout(cct, 20) << \"offset=\" << offset << \", \"\n+                 << \"length=\" << length << \", \"\n+                 << \"on_finish=\" << on_finish << dendl;\n+\n+  {\n+    RWLock::RLocker snap_locker(m_image_ctx.snap_lock);\n+    if (m_image_ctx.snap_id != CEPH_NOSNAP || m_image_ctx.read_only) {\n+      on_finish->complete(-EROFS);\n+      return;\n+    }\n+  }\n+\n+  if (m_image_ctx.persistent_cache_enabled) {\n+    m_image_cache.aio_discard(offset, length, skip_partial_discard, on_finish);\n+  } else {\n+    m_image_writeback.aio_discard(offset, length, skip_partial_discard, on_finish);\n+  }\n+  /*\n+  if (!is_block_aligned({{offset, length}})) {\n+    // For clients that don't use LBA extents, re-align the discard request\n+    // to work with the cache\n+    ldout(cct, 20) << \"aligning discard to block size\" << dendl;\n+\n+    // TODO: for non-aligned extents, invalidate the associated block-aligned\n+    // regions in the cache (if any), send the aligned extents to the cache\n+    // and the un-aligned extents directly to back to librbd\n+  }\n+\n+  // TODO invalidate discard blocks until writethrough/back support added\n+  C_Gather *ctx = new C_Gather(cct, on_finish);\n+  Context *invalidate_done_ctx = ctx->new_sub();\n+\n+  m_image_writeback.aio_discard(offset, length, skip_partial_discard, ctx->new_sub());\n+\n+  ctx->activate();\n+\n+  Context *invalidate_ctx = new FunctionContext(\n+    [this, offset, length, invalidate_done_ctx](int r) {\n+      invalidate({{offset, length}}, invalidate_done_ctx);\n+    });\n+  flush(invalidate_ctx);\n+  */\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::aio_flush(Context *on_finish) {\n+  CephContext *cct = m_image_ctx.cct;\n+  ldout(cct, 20) << \"on_finish=\" << on_finish << dendl;\n+\n+  {\n+    RWLock::RLocker snap_locker(m_image_ctx.snap_lock);\n+    if (m_image_ctx.snap_id != CEPH_NOSNAP || m_image_ctx.read_only) {\n+      on_finish->complete(-EROFS);\n+      return;\n+    }\n+  }\n+\n+  if (!m_flush_seen) {\n+    Mutex::Locker locker(m_lock);\n+    if (!m_flush_seen) {\n+      ldout(cct, 5) << \"flush seen\" << dendl;\n+      m_flush_seen = true;\n+      if (!m_persist_on_flush && m_persist_on_write_until_flush) {\n+\tm_persist_on_flush = true;\n+\tldout(cct, 5) << \"now persisting on flush\" << dendl;\n+      }\n+    }\n+  }\n+  \n+  if (m_image_ctx.persistent_cache_enabled) {\n+    m_image_cache.aio_flush(on_finish);\n+  } else {\n+    m_image_writeback.aio_flush(on_finish);\n+  }\n+  /*\n+  Context *ctx = new FunctionContext(\n+    [this, on_finish](int r) {\n+      if (r < 0) {\n+        on_finish->complete(r);\n+      }\n+      m_image_writeback.aio_flush(on_finish);\n+    });\n+\n+  flush(ctx);\n+  */\n+}\n+\n+template <typename I>\n+void ReplicatedWriteLog<I>::aio_writesame(uint64_t offset, uint64_t length,\n+                                      bufferlist&& bl, int fadvise_flags,\n+                                      Context *on_finish) {\n+  CephContext *cct = m_image_ctx.cct;\n+  ldout(cct, 20) << \"offset=\" << offset << \", \"\n+                 << \"length=\" << length << \", \"\n+                 << \"data_len=\" << bl.length() << \", \"\n+                 << \"on_finish=\" << on_finish << dendl;\n+  {\n+    RWLock::RLocker snap_locker(m_image_ctx.snap_lock);\n+    if (m_image_ctx.snap_id != CEPH_NOSNAP || m_image_ctx.read_only) {\n+      on_finish->complete(-EROFS);\n+      return;\n+    }\n+  }\n+\n+  if (m_image_ctx.persistent_cache_enabled) {","path":"src/librbd/cache/ReplicatedWriteLog.cc","position":1129,"original_position":1221,"commit_id":"b1212ed12a036c2b89852058bfcaba44c7c11b15","original_commit_id":"b1212ed12a036c2b89852058bfcaba44c7c11b15","user":{"login":"dillaman","id":4739042,"avatar_url":"https://avatars0.githubusercontent.com/u/4739042?v=4","gravatar_id":"","url":"https://api.github.com/users/dillaman","html_url":"https://github.com/dillaman","followers_url":"https://api.github.com/users/dillaman/followers","following_url":"https://api.github.com/users/dillaman/following{/other_user}","gists_url":"https://api.github.com/users/dillaman/gists{/gist_id}","starred_url":"https://api.github.com/users/dillaman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dillaman/subscriptions","organizations_url":"https://api.github.com/users/dillaman/orgs","repos_url":"https://api.github.com/users/dillaman/repos","events_url":"https://api.github.com/users/dillaman/events{/privacy}","received_events_url":"https://api.github.com/users/dillaman/received_events","type":"User","site_admin":false},"body":"Similar use-case as compare-and-write and should probably just be a passthrough or initially flagged as non-supported by returning an error.","created_at":"2018-02-12T17:11:01Z","updated_at":"2018-02-12T17:31:56Z","html_url":"https://github.com/ceph/ceph/pull/20375#discussion_r167623326","pull_request_url":"https://api.github.com/repos/ceph/ceph/pulls/20375","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/ceph/ceph/pulls/comments/167623326"},"html":{"href":"https://github.com/ceph/ceph/pull/20375#discussion_r167623326"},"pull_request":{"href":"https://api.github.com/repos/ceph/ceph/pulls/20375"}}},"pull_request":{"url":"https://api.github.com/repos/ceph/ceph/pulls/20375","id":168085984,"html_url":"https://github.com/ceph/ceph/pull/20375","diff_url":"https://github.com/ceph/ceph/pull/20375.diff","patch_url":"https://github.com/ceph/ceph/pull/20375.patch","issue_url":"https://api.github.com/repos/ceph/ceph/issues/20375","number":20375,"state":"open","locked":false,"title":"[DNM] RFC: Write back cache for RBD","user":{"login":"sdpeters","id":21087154,"avatar_url":"https://avatars1.githubusercontent.com/u/21087154?v=4","gravatar_id":"","url":"https://api.github.com/users/sdpeters","html_url":"https://github.com/sdpeters","followers_url":"https://api.github.com/users/sdpeters/followers","following_url":"https://api.github.com/users/sdpeters/following{/other_user}","gists_url":"https://api.github.com/users/sdpeters/gists{/gist_id}","starred_url":"https://api.github.com/users/sdpeters/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sdpeters/subscriptions","organizations_url":"https://api.github.com/users/sdpeters/orgs","repos_url":"https://api.github.com/users/sdpeters/repos","events_url":"https://api.github.com/users/sdpeters/events{/privacy}","received_events_url":"https://api.github.com/users/sdpeters/received_events","type":"User","site_admin":false},"body":"# Replicated Write Log (RWL) for RBD\r\n\r\n**Do not merge. RFC only.**\r\n\r\nThis patch introduces the Replicated Write Log (RWL) for RBD.  RWL is a write-back cache for RBD. RWL works best in byte addressable persistent memory. RWL supports replication, ideally over RDMA.\r\n\r\nThis caching solution is derived from one of the previous Intel Ceph caching branches (wip-ilbrbd-cache), but has different objectives. This branch targets write tail latency. Our goal is 99.99% < 1mS for a rate limited (~10K IOPS) RBD client, and we seem to be easily meeting that with this prototype code. \r\n \r\nThis cache is structured as a log. Writes are appended to the end of the log. We flush these writes from the old end of the log, then retire the flushed entries from the log when it needs the space. \r\n \r\nWhat's working in this branch is basic writes and reads. One of the librbd unit tests (tweaked to skip comp-and-write, etc. if this cache is enabled) passes. I've run fio on it to get latency measurements, but flushing to RBD isn't implemented yet so that eventually fails when the log fills. The rbd aio_flush() call is implemented, and I have a tweaked version of fio that will call it when barriers are applied to RBD volumes. The cache is designed to exploit the transition to persist-on-flush to defer and aggregate the persistence operations required to append writes.\r\n \r\nThis cache can be read from, but isn't expected to be used as a read cache. It will not fill itself from RBD on a miss, just pass the miss extents down (and reassemble the read result from that and the hit extents when the lower read completes). There are several competing RBD read caching initiatives. This cache stacks on top of one of those, reading and flushing through it. When a clear winner among the read caches emerges, we can consider integrating these two caches if that makes sense (I expect it may not, because some copying/moving will probably always be necessary to coalesce the writes).\r\n \r\nThis write cache is intended to perform best with byte addressable persistent memory, but will work on SSDs. It uses libpmemobj from the PMDK (NVML) to achieve persistence, atomicity, and free space management. When used on SSDs, the persistence steps produce msync() calls. This code attempts to aggregate the persistence phase of as many writes as are ready for it, and does that on selected threads. This design will produce writes that are smaller than ideal for an SSD, but we should be able to aggregate them in persist-on-flush mode (more below). \r\n \r\nInside the pmem pool (whether that's really on 3DXPoint or an SSD) the log is laid out as a circular buffer of small write log entries, and a head and tail index. That ring buffer is allocated when the cache is initialized, and on subsequent opens is located from the pointer (actually a pmemOID) in the root object. This ring buffer contains the metadata for the unflushed sequence of writes (<= 64 bytes/write). Each of the items in the ring buffer contains an OID for the buffer containing the write data. The write buffers are variable sized.\r\n \r\nA single write reserves a new buffer in the pool, copies the data to that address, persists the data (msync() from a finisher), allocates the ring buffer slot, copies that structure, persists that ring slot (msync() from a different finisher), then performs a libpmemobj transaction to atomically update the ring head index and persist the allocation of the reserved buffer it has already written to (another pair of msync()s form a third finisher). At that point if we crash these writes are visible. Until here, these writes fail (did not occur) on a crash (and the buffer allocation also does not occur).\r\n \r\nAt IO depth >1, the buffer allocations and copies above all run concurrently, and the data persist finisher may persist several of them at once if they're adjacent. The ring buffer slots are always adjacent, so the copy and persist of these is easily aggregated into a single persist operation. Likewise, the ring head pointer and allocation persists are all done in a single transaction which needs one msync() for the undo log, and another to commit.\r\n \r\nIn persist-on-write mode, user writes aren't completed until the write is visible (the head pointer and alloc persist transaction completes). In persist-on-flush mode, user writes complete as soon as the data is copied from their buffer into the reserved buffer in the pool, and before the persist step for that buffer. For this reason I tend to disable the rbd option that forces all writes t the workq. It appears to be unsafe to call the users callback before the write call completes in this API, so though we may mark their aio completed before we return the callback is still pushed to another thread. We could fix that, but I think we'd need an API change to indicate the callback was safe from the dispatching thread. \r\n \r\nSo, in persist-on-flush mode we may have more opportunities to aggregate persistence operations.\r\n \r\nIn persist-on-write mode aio_flush() is mostly a no-op. In persist-on-flush mode, it forces a sync point to be inserted in the write log and waits for that. A sync point is not appended until all prior writes complete to visibility.\r\n \r\nWhen 3DXpoint is used, the persist phase of a write may not do much. \r\n \r\nThe libpmemobj API also provides optional remote replication. This is working in our lab, but requires some library changes before it can be presented as a complete solution. Replication works over TCP (because librpmem uses libfabric, which supports verbs and plain old sockets). When 3DXPoint and RDMA is used, the replica write path is all hardware (no CPU on the replica until something fails). This is mostly invisible to the IO path code you see here. The persistence steps involve network communications which, like msync(), may block or poll. That's all hidden in libpmemobj. The same aggregation and threading strategy works there, so the only special case for replication is how the pool is opened and initialized.\r\n \r\nWhen replication is used, ordered write back is not required for consistency (because unflushed writes are not lost when one node or device fails). Users may still want OWB if they're using remote RBD replication, but we hope they won't. When OWB isn't needed, this cache is free to reorder what it flushes, and skip overwrites. In that case, it can reclaim cache space without flushing when data is overwritten (unmapped, etc.). Since write data buffers are separately allocated, these can be freed for log entries that were later completely overwritten. So if we can avoid OWB we think we'll flush less and flush it faster by choosing convenient flush write sizes and alignments.\r\n \r\nForced OWB can be made a configurable option. I expected to implement that as the initial flushing algorithm anyway, because it's simpler. It is not possible to switch to OWB from non-OWB, of course, because out of order flushing and/or in-place write coalescing may have already occurred.\r\n\r\n","created_at":"2018-02-08T21:01:57Z","updated_at":"2018-02-12T17:31:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":647438941,"url":"https://api.github.com/repos/ceph/ceph/labels/DNM","name":"DNM","color":"ff0000","default":false},{"id":144329703,"url":"https://api.github.com/repos/ceph/ceph/labels/rbd","name":"rbd","color":"40bfb2","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/ceph/ceph/pulls/20375/commits","review_comments_url":"https://api.github.com/repos/ceph/ceph/pulls/20375/comments","review_comment_url":"https://api.github.com/repos/ceph/ceph/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ceph/ceph/issues/20375/comments","statuses_url":"https://api.github.com/repos/ceph/ceph/statuses/b1212ed12a036c2b89852058bfcaba44c7c11b15","head":{"label":"sdpeters:wip-librbd-rwl","ref":"wip-librbd-rwl","sha":"b1212ed12a036c2b89852058bfcaba44c7c11b15","user":{"login":"sdpeters","id":21087154,"avatar_url":"https://avatars1.githubusercontent.com/u/21087154?v=4","gravatar_id":"","url":"https://api.github.com/users/sdpeters","html_url":"https://github.com/sdpeters","followers_url":"https://api.github.com/users/sdpeters/followers","following_url":"https://api.github.com/users/sdpeters/following{/other_user}","gists_url":"https://api.github.com/users/sdpeters/gists{/gist_id}","starred_url":"https://api.github.com/users/sdpeters/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sdpeters/subscriptions","organizations_url":"https://api.github.com/users/sdpeters/orgs","repos_url":"https://api.github.com/users/sdpeters/repos","events_url":"https://api.github.com/users/sdpeters/events{/privacy}","received_events_url":"https://api.github.com/users/sdpeters/received_events","type":"User","site_admin":false},"repo":{"id":120809807,"name":"ceph","full_name":"sdpeters/ceph","owner":{"login":"sdpeters","id":21087154,"avatar_url":"https://avatars1.githubusercontent.com/u/21087154?v=4","gravatar_id":"","url":"https://api.github.com/users/sdpeters","html_url":"https://github.com/sdpeters","followers_url":"https://api.github.com/users/sdpeters/followers","following_url":"https://api.github.com/users/sdpeters/following{/other_user}","gists_url":"https://api.github.com/users/sdpeters/gists{/gist_id}","starred_url":"https://api.github.com/users/sdpeters/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sdpeters/subscriptions","organizations_url":"https://api.github.com/users/sdpeters/orgs","repos_url":"https://api.github.com/users/sdpeters/repos","events_url":"https://api.github.com/users/sdpeters/events{/privacy}","received_events_url":"https://api.github.com/users/sdpeters/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sdpeters/ceph","description":"Ceph is a distributed object, block, and file storage platform ","fork":true,"url":"https://api.github.com/repos/sdpeters/ceph","forks_url":"https://api.github.com/repos/sdpeters/ceph/forks","keys_url":"https://api.github.com/repos/sdpeters/ceph/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sdpeters/ceph/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sdpeters/ceph/teams","hooks_url":"https://api.github.com/repos/sdpeters/ceph/hooks","issue_events_url":"https://api.github.com/repos/sdpeters/ceph/issues/events{/number}","events_url":"https://api.github.com/repos/sdpeters/ceph/events","assignees_url":"https://api.github.com/repos/sdpeters/ceph/assignees{/user}","branches_url":"https://api.github.com/repos/sdpeters/ceph/branches{/branch}","tags_url":"https://api.github.com/repos/sdpeters/ceph/tags","blobs_url":"https://api.github.com/repos/sdpeters/ceph/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sdpeters/ceph/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sdpeters/ceph/git/refs{/sha}","trees_url":"https://api.github.com/repos/sdpeters/ceph/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sdpeters/ceph/statuses/{sha}","languages_url":"https://api.github.com/repos/sdpeters/ceph/languages","stargazers_url":"https://api.github.com/repos/sdpeters/ceph/stargazers","contributors_url":"https://api.github.com/repos/sdpeters/ceph/contributors","subscribers_url":"https://api.github.com/repos/sdpeters/ceph/subscribers","subscription_url":"https://api.github.com/repos/sdpeters/ceph/subscription","commits_url":"https://api.github.com/repos/sdpeters/ceph/commits{/sha}","git_commits_url":"https://api.github.com/repos/sdpeters/ceph/git/commits{/sha}","comments_url":"https://api.github.com/repos/sdpeters/ceph/comments{/number}","issue_comment_url":"https://api.github.com/repos/sdpeters/ceph/issues/comments{/number}","contents_url":"https://api.github.com/repos/sdpeters/ceph/contents/{+path}","compare_url":"https://api.github.com/repos/sdpeters/ceph/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sdpeters/ceph/merges","archive_url":"https://api.github.com/repos/sdpeters/ceph/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sdpeters/ceph/downloads","issues_url":"https://api.github.com/repos/sdpeters/ceph/issues{/number}","pulls_url":"https://api.github.com/repos/sdpeters/ceph/pulls{/number}","milestones_url":"https://api.github.com/repos/sdpeters/ceph/milestones{/number}","notifications_url":"https://api.github.com/repos/sdpeters/ceph/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sdpeters/ceph/labels{/name}","releases_url":"https://api.github.com/repos/sdpeters/ceph/releases{/id}","deployments_url":"https://api.github.com/repos/sdpeters/ceph/deployments","created_at":"2018-02-08T19:49:39Z","updated_at":"2018-02-08T19:50:01Z","pushed_at":"2018-02-08T19:54:19Z","git_url":"git://github.com/sdpeters/ceph.git","ssh_url":"git@github.com:sdpeters/ceph.git","clone_url":"https://github.com/sdpeters/ceph.git","svn_url":"https://github.com/sdpeters/ceph","homepage":"http://ceph.com","size":267395,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ceph:master","ref":"master","sha":"7a5777183ec767a773f9579386cdac4034ef8727","user":{"login":"ceph","id":1015767,"avatar_url":"https://avatars3.githubusercontent.com/u/1015767?v=4","gravatar_id":"","url":"https://api.github.com/users/ceph","html_url":"https://github.com/ceph","followers_url":"https://api.github.com/users/ceph/followers","following_url":"https://api.github.com/users/ceph/following{/other_user}","gists_url":"https://api.github.com/users/ceph/gists{/gist_id}","starred_url":"https://api.github.com/users/ceph/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ceph/subscriptions","organizations_url":"https://api.github.com/users/ceph/orgs","repos_url":"https://api.github.com/users/ceph/repos","events_url":"https://api.github.com/users/ceph/events{/privacy}","received_events_url":"https://api.github.com/users/ceph/received_events","type":"Organization","site_admin":false},"repo":{"id":2310495,"name":"ceph","full_name":"ceph/ceph","owner":{"login":"ceph","id":1015767,"avatar_url":"https://avatars3.githubusercontent.com/u/1015767?v=4","gravatar_id":"","url":"https://api.github.com/users/ceph","html_url":"https://github.com/ceph","followers_url":"https://api.github.com/users/ceph/followers","following_url":"https://api.github.com/users/ceph/following{/other_user}","gists_url":"https://api.github.com/users/ceph/gists{/gist_id}","starred_url":"https://api.github.com/users/ceph/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ceph/subscriptions","organizations_url":"https://api.github.com/users/ceph/orgs","repos_url":"https://api.github.com/users/ceph/repos","events_url":"https://api.github.com/users/ceph/events{/privacy}","received_events_url":"https://api.github.com/users/ceph/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ceph/ceph","description":"Ceph is a distributed object, block, and file storage platform ","fork":false,"url":"https://api.github.com/repos/ceph/ceph","forks_url":"https://api.github.com/repos/ceph/ceph/forks","keys_url":"https://api.github.com/repos/ceph/ceph/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ceph/ceph/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ceph/ceph/teams","hooks_url":"https://api.github.com/repos/ceph/ceph/hooks","issue_events_url":"https://api.github.com/repos/ceph/ceph/issues/events{/number}","events_url":"https://api.github.com/repos/ceph/ceph/events","assignees_url":"https://api.github.com/repos/ceph/ceph/assignees{/user}","branches_url":"https://api.github.com/repos/ceph/ceph/branches{/branch}","tags_url":"https://api.github.com/repos/ceph/ceph/tags","blobs_url":"https://api.github.com/repos/ceph/ceph/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ceph/ceph/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ceph/ceph/git/refs{/sha}","trees_url":"https://api.github.com/repos/ceph/ceph/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ceph/ceph/statuses/{sha}","languages_url":"https://api.github.com/repos/ceph/ceph/languages","stargazers_url":"https://api.github.com/repos/ceph/ceph/stargazers","contributors_url":"https://api.github.com/repos/ceph/ceph/contributors","subscribers_url":"https://api.github.com/repos/ceph/ceph/subscribers","subscription_url":"https://api.github.com/repos/ceph/ceph/subscription","commits_url":"https://api.github.com/repos/ceph/ceph/commits{/sha}","git_commits_url":"https://api.github.com/repos/ceph/ceph/git/commits{/sha}","comments_url":"https://api.github.com/repos/ceph/ceph/comments{/number}","issue_comment_url":"https://api.github.com/repos/ceph/ceph/issues/comments{/number}","contents_url":"https://api.github.com/repos/ceph/ceph/contents/{+path}","compare_url":"https://api.github.com/repos/ceph/ceph/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ceph/ceph/merges","archive_url":"https://api.github.com/repos/ceph/ceph/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ceph/ceph/downloads","issues_url":"https://api.github.com/repos/ceph/ceph/issues{/number}","pulls_url":"https://api.github.com/repos/ceph/ceph/pulls{/number}","milestones_url":"https://api.github.com/repos/ceph/ceph/milestones{/number}","notifications_url":"https://api.github.com/repos/ceph/ceph/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ceph/ceph/labels{/name}","releases_url":"https://api.github.com/repos/ceph/ceph/releases{/id}","deployments_url":"https://api.github.com/repos/ceph/ceph/deployments","created_at":"2011-09-01T21:41:26Z","updated_at":"2018-02-12T15:55:50Z","pushed_at":"2018-02-12T17:18:46Z","git_url":"git://github.com/ceph/ceph.git","ssh_url":"git@github.com:ceph/ceph.git","clone_url":"https://github.com/ceph/ceph.git","svn_url":"https://github.com/ceph/ceph","homepage":"http://ceph.com","size":267965,"stargazers_count":4113,"watchers_count":4113,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2310,"mirror_url":null,"archived":false,"open_issues_count":618,"license":null,"forks":2310,"open_issues":618,"watchers":4113,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ceph/ceph/pulls/20375"},"html":{"href":"https://github.com/ceph/ceph/pull/20375"},"issue":{"href":"https://api.github.com/repos/ceph/ceph/issues/20375"},"comments":{"href":"https://api.github.com/repos/ceph/ceph/issues/20375/comments"},"review_comments":{"href":"https://api.github.com/repos/ceph/ceph/pulls/20375/comments"},"review_comment":{"href":"https://api.github.com/repos/ceph/ceph/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ceph/ceph/pulls/20375/commits"},"statuses":{"href":"https://api.github.com/repos/ceph/ceph/statuses/b1212ed12a036c2b89852058bfcaba44c7c11b15"}},"author_association":"NONE"}}
{"actor":{"display_login":"dillaman"}}