X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=ceph%2Fsrc%2Fcls%2Frgw%2Fcls_rgw.cc;h=17a618053a61effcb2185ba148fa93bf411b7bbe;hb=c07f9fc5a4f48397831383549fb0482b93480643;hp=9af96fe6ab4852fb4f9a706261f0fb3e3980a795;hpb=9439ae556f035e65c9c107ae13ddd09457dbbecd;p=ceph.git diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index 9af96fe6a..17a618053 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -145,9 +145,9 @@ static int log_index_operation(cls_method_context_t hctx, cls_rgw_obj_key& obj_k * read list of objects, skips objects in the ugly namespace */ static int get_obj_vals(cls_method_context_t hctx, const string& start, const string& filter_prefix, - int num_entries, map *pkeys) + int num_entries, map *pkeys, bool *pmore) { - int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys); + int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys, pmore); if (ret < 0) return ret; @@ -183,7 +183,7 @@ static int get_obj_vals(cls_method_context_t hctx, const string& start, const st string new_start = c; /* now get some more keys */ - ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys); + ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys, pmore); if (ret < 0) return ret; @@ -405,10 +405,11 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) string start_key; encode_list_index_key(hctx, op.start_obj, &start_key); bool done = false; - uint32_t left_to_read = op.num_entries + 1; + uint32_t left_to_read = op.num_entries; + bool more; do { - rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys); + rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys, &more); if (rc < 0) return rc; @@ -458,8 +459,7 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } } while (left_to_read > 0 && !done); - ret.is_truncated = (left_to_read == 0) && /* we found more entries than we were requested, meaning response is truncated */ - !done; + ret.is_truncated = more && !done; ::encode(ret, *out); return 0; @@ -482,9 +482,10 @@ static int check_index(cls_method_context_t hctx, struct rgw_bucket_dir_header * #define CHECK_CHUNK_SIZE 1000 bool done = false; + bool more; do { - rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys); + rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys, &more); if (rc < 0) return rc; @@ -689,7 +690,7 @@ int rgw_bucket_prepare_op(cls_method_context_t hctx, bufferlist *in, bufferlist return rc; } - if (op.log_op) { + if (op.log_op && !header.syncstopped) { rc = log_index_operation(hctx, op.key, op.op, op.tag, entry.meta.mtime, entry.ver, info.state, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); if (rc < 0) @@ -846,7 +847,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist bufferlist op_bl; if (cancel) { - if (op.log_op) { + if (op.log_op && !header.syncstopped) { rc = log_index_operation(hctx, op.key, op.op, op.tag, entry.meta.mtime, entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); if (rc < 0) @@ -908,7 +909,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist break; } - if (op.log_op) { + if (op.log_op && !header.syncstopped) { rc = log_index_operation(hctx, op.key, op.op, op.tag, entry.meta.mtime, entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); if (rc < 0) @@ -933,7 +934,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist remove_entry.key.name.c_str(), remove_entry.key.instance.c_str(), remove_entry.meta.category); unaccount_entry(header, remove_entry); - if (op.log_op) { + if (op.log_op && !header.syncstopped) { rc = log_index_operation(hctx, remove_key, CLS_RGW_OP_DEL, op.tag, remove_entry.meta.mtime, remove_entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); if (rc < 0) @@ -1157,9 +1158,10 @@ public: get_list_index_key(instance_entry, &list_idx); /* this is the current head, need to update! */ map keys; + bool more; string filter = key.name; /* list key starts with key name, filter it to avoid a case where we cross to different namespace */ - int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys); + int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys, &more); if (ret < 0) { return ret; } @@ -1520,7 +1522,7 @@ static int rgw_bucket_link_olh(cls_method_context_t hctx, bufferlist *in, buffer return ret; } - if (op.log_op) { + if (op.log_op && !header.syncstopped) { rgw_bucket_dir_entry& entry = obj.get_dir_entry(); rgw_bucket_entry_ver ver; @@ -1676,7 +1678,7 @@ static int rgw_bucket_unlink_instance(cls_method_context_t hctx, bufferlist *in, return ret; } - if (op.log_op) { + if (op.log_op && !header.syncstopped) { rgw_bucket_entry_ver ver; ver.epoch = (op.olh_epoch ? op.olh_epoch : olh.get_epoch()); @@ -1944,7 +1946,7 @@ int rgw_dir_suggest_changes(cls_method_context_t hctx, bufferlist *in, bufferlis ret = cls_cxx_map_remove_key(hctx, cur_change_key); if (ret < 0) return ret; - if (log_op && cur_disk.exists) { + if (log_op && cur_disk.exists && !header.syncstopped) { ret = log_index_operation(hctx, cur_disk.key, CLS_RGW_OP_DEL, cur_disk.tag, cur_disk.meta.mtime, cur_disk.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, 0, NULL, NULL, NULL); if (ret < 0) { @@ -1967,7 +1969,7 @@ int rgw_dir_suggest_changes(cls_method_context_t hctx, bufferlist *in, bufferlis ret = cls_cxx_map_set_val(hctx, cur_change_key, &cur_state_bl); if (ret < 0) return ret; - if (log_op) { + if (log_op && !header.syncstopped) { ret = log_index_operation(hctx, cur_change.key, CLS_RGW_OP_ADD, cur_change.tag, cur_change.meta.mtime, cur_change.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, 0, NULL, NULL, NULL); if (ret < 0) { @@ -2260,7 +2262,7 @@ static int rgw_bi_put_op(cls_method_context_t hctx, bufferlist *in, bufferlist * } static int list_plain_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { string filter = name; string start_key = marker; @@ -2270,59 +2272,52 @@ static int list_plain_entries(cls_method_context_t hctx, const string& name, con int count = 0; map keys; - do { - if (count >= (int)max) { + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max, &keys, pmore); + if (ret < 0) { + return ret; + } + + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + if (iter->first >= end_key) { + /* past the end of plain namespace */ return count; } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_GET_NUM_KEYS, &keys); - if (ret < 0) { - return ret; - } - - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - if (iter->first >= end_key) { - /* past the end of plain namespace */ - return count; - } - rgw_cls_bi_entry entry; - entry.type = PlainIdx; - entry.idx = iter->first; - entry.data = iter->second; + rgw_cls_bi_entry entry; + entry.type = PlainIdx; + entry.idx = iter->first; + entry.data = iter->second; - bufferlist::iterator biter = entry.data.begin(); + bufferlist::iterator biter = entry.data.begin(); - rgw_bucket_dir_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__); - return -EIO; - } + rgw_bucket_dir_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__); + return -EIO; + } - CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); + CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); - if (!name.empty() && e.key.name != name) { - return count; - } + if (!name.empty() && e.key.name != name) { + return count; + } - entries->push_back(entry); - count++; - if (count >= (int)max) { - return count; - } - start_key = entry.idx; + entries->push_back(entry); + count++; + if (count >= (int)max) { + return count; } - } while (!keys.empty()); + start_key = entry.idx; + } return count; } static int list_instance_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { cls_rgw_obj_key key(name); string first_instance_idx; @@ -2341,66 +2336,63 @@ static int list_instance_entries(cls_method_context_t hctx, const string& name, } int count = 0; map keys; - bool started = true; - do { - if (count >= (int)max) { - return count; - } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret; - if (started) { - ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]); - if (ret == -ENOENT) { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } - started = false; - } else { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } + bufferlist k; + int ret = cls_cxx_map_get_val(hctx, start_key, &k); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + bool found_first = (ret == 0); + if (found_first) { + --max; + } + if (max > 0) { + ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore); CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size()); if (ret < 0) { return ret; } + } + if (found_first) { + keys[start_key].claim(k); + } - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - rgw_cls_bi_entry entry; - entry.type = InstanceIdx; - entry.idx = iter->first; - entry.data = iter->second; - - if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { - return count; - } + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + rgw_cls_bi_entry entry; + entry.type = InstanceIdx; + entry.idx = iter->first; + entry.data = iter->second; - CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); + if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + return count; + } - bufferlist::iterator biter = entry.data.begin(); + CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); - rgw_bucket_dir_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); - return -EIO; - } + bufferlist::iterator biter = entry.data.begin(); - if (!name.empty() && e.key.name != name) { - return count; - } + rgw_bucket_dir_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); + return -EIO; + } - entries->push_back(entry); - count++; - start_key = entry.idx; + if (!name.empty() && e.key.name != name) { + return count; } - } while (!keys.empty()); + + entries->push_back(entry); + count++; + start_key = entry.idx; + } return count; } static int list_olh_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { cls_rgw_obj_key key(name); string first_instance_idx; @@ -2419,60 +2411,59 @@ static int list_olh_entries(cls_method_context_t hctx, const string& name, const } int count = 0; map keys; - bool started = true; - do { - if (count >= (int)max) { - return count; - } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret; - if (started) { - ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]); - if (ret == -ENOENT) { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } - started = false; - } else { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } + int ret; + bufferlist k; + ret = cls_cxx_map_get_val(hctx, start_key, &k); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + bool found_first = (ret == 0); + if (found_first) { + --max; + } + if (max > 0) { + ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore); CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size()); if (ret < 0) { return ret; } + } - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - rgw_cls_bi_entry entry; - entry.type = OLHIdx; - entry.idx = iter->first; - entry.data = iter->second; + if (found_first) { + keys[start_key].claim(k); + } - if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { - return count; - } + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + rgw_cls_bi_entry entry; + entry.type = OLHIdx; + entry.idx = iter->first; + entry.data = iter->second; - CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); + if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + return count; + } - bufferlist::iterator biter = entry.data.begin(); + CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); - rgw_bucket_olh_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); - return -EIO; - } + bufferlist::iterator biter = entry.data.begin(); - if (!name.empty() && e.key.name != name) { - return count; - } + rgw_bucket_olh_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); + return -EIO; + } - entries->push_back(entry); - count++; - start_key = entry.idx; + if (!name.empty() && e.key.name != name) { + return count; } - } while (!keys.empty()); + + entries->push_back(entry); + count++; + start_key = entry.idx; + } return count; } @@ -2493,9 +2484,10 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist string filter = op.name; #define MAX_BI_LIST_ENTRIES 1000 - int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES) + 1; /* one extra entry for identifying truncation */ + int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES); string start_key = op.marker; - int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries); + bool more; + int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries, &more); if (ret < 0) { CLS_LOG(0, "ERROR: %s(): list_plain_entries retured ret=%d", __func__, ret); return ret; @@ -2504,23 +2496,27 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist CLS_LOG(20, "found %d plain entries", count); - ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries); - if (ret < 0) { - CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret); - return ret; + if (!more) { + ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more); + if (ret < 0) { + CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret); + return ret; + } + + count += ret; } - count += ret; + if (!more) { + ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more); + if (ret < 0) { + CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret); + return ret; + } - ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries); - if (ret < 0) { - CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret); - return ret; + count += ret; } - count += ret; - - op_ret.is_truncated = (count >= max); + op_ret.is_truncated = (count >= max) || more; while (count >= max) { op_ret.entries.pop_back(); count--; @@ -2582,45 +2578,40 @@ static int bi_log_iterate_entries(cls_method_context_t hctx, const string& marke string filter; - do { -#define BI_NUM_KEYS 128 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_NUM_KEYS, &keys); - if (ret < 0) - return ret; + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - rgw_bi_log_entry e; + uint32_t num_keys = keys.size(); - CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length()); + for (; iter != keys.end(); ++iter,++i) { + const string& key = iter->first; + rgw_bi_log_entry e; - if (key.compare(end_key) > 0) - return 0; + CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length()); - ret = bi_log_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (key.compare(end_key) > 0) { + key_iter = key; + return 0; + } - if (max_entries && (i >= max_entries)) { - if (truncated) - *truncated = true; - key_iter = key; - return 0; - } + ret = bi_log_record_decode(iter->second, e); + if (ret < 0) + return ret; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; - i++; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; + if (i == num_keys - 1) { + key_iter = key; } - --iter; - start_key = iter->first; - } while (true); + } + return 0; } @@ -2728,6 +2719,74 @@ static int rgw_bi_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlist return 0; } +static int rgw_bi_log_resync(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + struct rgw_bucket_dir_header header; + int rc = read_bucket_header(hctx, &header); + if (rc < 0) { + CLS_LOG(1, "ERROR: rgw_bucket_complete_op(): failed to read header\n"); + return rc; + } + + bufferlist bl; + + struct rgw_bi_log_entry entry; + + entry.timestamp = real_clock::now(); + entry.op = RGWModifyOp::CLS_RGW_OP_RESYNC; + entry.state = RGWPendingState::CLS_RGW_STATE_COMPLETE; + + string key; + bi_log_index_key(hctx, key, entry.id, header.ver); + + ::encode(entry, bl); + + if (entry.id > header.max_marker) + header.max_marker = entry.id; + + header.syncstopped = false; + + rc = cls_cxx_map_set_val(hctx, key, &bl); + if (rc < 0) + return rc; + + return write_bucket_header(hctx, &header); +} + +static int rgw_bi_log_stop(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + struct rgw_bucket_dir_header header; + int rc = read_bucket_header(hctx, &header); + if (rc < 0) { + CLS_LOG(1, "ERROR: rgw_bucket_complete_op(): failed to read header\n"); + return rc; + } + + bufferlist bl; + + struct rgw_bi_log_entry entry; + + entry.timestamp = real_clock::now(); + entry.op = RGWModifyOp::CLS_RGW_OP_SYNCSTOP; + entry.state = RGWPendingState::CLS_RGW_STATE_COMPLETE; + + string key; + bi_log_index_key(hctx, key, entry.id, header.ver); + + ::encode(entry, bl); + + if (entry.id > header.max_marker) + header.max_marker = entry.id; + header.syncstopped = true; + + rc = cls_cxx_map_set_val(hctx, key, &bl); + if (rc < 0) + return rc; + + return write_bucket_header(hctx, &header); +} + + static void usage_record_prefix_by_time(uint64_t epoch, string& key) { char buf[32]; @@ -2862,58 +2921,53 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 start_key = key_iter; } - do { - CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); - int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, NUM_KEYS, &keys); - if (ret < 0) - return ret; + CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); + int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - rgw_usage_log_entry e; + uint32_t num_keys = keys.size(); - if (!by_user && key.compare(end_key) >= 0) { - CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); - return 0; - } + for (; iter != keys.end(); ++iter,++i) { + const string& key = iter->first; + rgw_usage_log_entry e; - if (by_user && key.compare(0, user_key.size(), user_key) != 0) { - CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); - return 0; - } + if (!by_user && key.compare(end_key) >= 0) { + CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + return 0; + } - ret = usage_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (by_user && key.compare(0, user_key.size(), user_key) != 0) { + CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + return 0; + } - if (e.epoch < start) - continue; + ret = usage_record_decode(iter->second, e); + if (ret < 0) + return ret; - /* keys are sorted by epoch, so once we're past end we're done */ - if (e.epoch >= end) - return 0; + if (e.epoch < start) + continue; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; + /* keys are sorted by epoch, so once we're past end we're done */ + if (e.epoch >= end) + return 0; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; - i++; - if (max_entries && (i > max_entries)) { - CLS_LOG(20, "usage_iterate_range reached max_entries (%d), done", max_entries); - *truncated = true; - key_iter = key; - return 0; - } + + if (i == num_keys - 1) { + key_iter = key; + return 0; } - --iter; - start_key = iter->first; - } while (true); + } return 0; } @@ -2999,7 +3053,9 @@ int rgw_user_usage_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlis } string iter; - ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, 0, NULL, usage_log_trim_cb, NULL); + bool more; +#define MAX_USAGE_TRIM_ENTRIES 128 + ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, MAX_USAGE_TRIM_ENTRIES, &more, usage_log_trim_cb, NULL); if (ret < 0) return ret; @@ -3197,50 +3253,42 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, b string filter; - do { -#define GC_NUM_KEYS 32 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, GC_NUM_KEYS, &keys); - if (ret < 0) - return ret; + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - cls_rgw_gc_obj_info e; + uint32_t num_keys = keys.size(); - CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); + for (; iter != keys.end(); ++iter, ++i) { + const string& key = iter->first; + cls_rgw_gc_obj_info e; - if (!end_key.empty() && key.compare(end_key) >= 0) - return 0; + CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); - if (!key_in_index(key, GC_OBJ_TIME_INDEX)) - return 0; + if (!end_key.empty() && key.compare(end_key) >= 0) + return 0; - ret = gc_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (!key_in_index(key, GC_OBJ_TIME_INDEX)) + return 0; - if (max_entries && (i >= max_entries)) { - if (truncated) - *truncated = true; - --iter; - key_iter = iter->first; - return 0; - } + ret = gc_record_decode(iter->second, e); + if (ret < 0) + return ret; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; - i++; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; + if (i == num_keys - 1) { + key_iter = key; } - --iter; - start_key = iter->first; - } while (true); + } + return 0; } @@ -3274,7 +3322,8 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist } cls_rgw_gc_list_ret op_ret; - int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, +#define GC_LIST_ENTRIES_DEFAULT 128 + int ret = gc_list_entries(hctx, op.marker, (op.max ? op.max : GC_LIST_ENTRIES_DEFAULT), op.expired_only, op_ret.entries, &op_ret.truncated, op_ret.next_marker); if (ret < 0) return ret; @@ -3384,7 +3433,8 @@ static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx, bufferlist *in, map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals); + bool more; + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals, &more); if (ret < 0) return ret; map::iterator it; @@ -3419,7 +3469,7 @@ static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, bu bufferlist::iterator iter; map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; @@ -3524,8 +3574,8 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis string filter_prefix; #define MAX_RESHARD_LIST_ENTRIES 1000 /* one extra entry for identifying truncation */ - int32_t max = (op.max < MAX_RESHARD_LIST_ENTRIES ? op.max : MAX_RESHARD_LIST_ENTRIES) + 1; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals); + int32_t max = (op.max && (op.max < MAX_RESHARD_LIST_ENTRIES) ? op.max : MAX_RESHARD_LIST_ENTRIES); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; @@ -3541,7 +3591,6 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis } op_ret.entries.push_back(entry); } - op_ret.is_truncated = op.max && (vals.size() > op.max); ::encode(op_ret, *out); return 0; } @@ -3747,6 +3796,8 @@ CLS_INIT(rgw) cls_method_handle_t h_rgw_bi_put_op; cls_method_handle_t h_rgw_bi_list_op; cls_method_handle_t h_rgw_bi_log_list_op; + cls_method_handle_t h_rgw_bi_log_resync_op; + cls_method_handle_t h_rgw_bi_log_stop_op; cls_method_handle_t h_rgw_dir_suggest_changes; cls_method_handle_t h_rgw_user_usage_log_add; cls_method_handle_t h_rgw_user_usage_log_read; @@ -3800,6 +3851,9 @@ CLS_INIT(rgw) cls_register_cxx_method(h_class, RGW_BI_LOG_TRIM, CLS_METHOD_RD | CLS_METHOD_WR, rgw_bi_log_trim, &h_rgw_bi_log_list_op); cls_register_cxx_method(h_class, RGW_DIR_SUGGEST_CHANGES, CLS_METHOD_RD | CLS_METHOD_WR, rgw_dir_suggest_changes, &h_rgw_dir_suggest_changes); + cls_register_cxx_method(h_class, "bi_log_resync", CLS_METHOD_RD | CLS_METHOD_WR, rgw_bi_log_resync, &h_rgw_bi_log_resync_op); + cls_register_cxx_method(h_class, "bi_log_stop", CLS_METHOD_RD | CLS_METHOD_WR, rgw_bi_log_stop, &h_rgw_bi_log_stop_op); + /* usage logging */ cls_register_cxx_method(h_class, RGW_USER_USAGE_LOG_ADD, CLS_METHOD_RD | CLS_METHOD_WR, rgw_user_usage_log_add, &h_rgw_user_usage_log_add); cls_register_cxx_method(h_class, RGW_USER_USAGE_LOG_READ, CLS_METHOD_RD, rgw_user_usage_log_read, &h_rgw_user_usage_log_read);