Skip to content

Commit eb26360

Browse files
yuriwyuriw
authored
Merge pull request ceph#15807 from zhangsw/feature-lifecycle-date
rgw: S3 lifecycle now supports expiration date Reviewed-by: Daniel Gryniewicz <[email protected]>
2 parents 7aabdc0 + 26a2051 commit eb26360

File tree

4 files changed

+125
-41
lines changed

4 files changed

+125
-41
lines changed

‎src/rgw/rgw_lc.cc

+53-24
Original file line numberDiff line numberDiff line change
@@ -27,21 +27,15 @@ const char* LC_STATUS[] = {
2727
using namespace std;
2828
using namespace librados;
2929

30-
bool LCRule::validate()
30+
bool LCRule::valid()
3131
{
3232
if (id.length() > MAX_ID_LEN) {
3333
return false;
3434
}
3535
else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty() && !dm_expiration) {
3636
return false;
3737
}
38-
else if (!expiration.empty() && expiration.get_days() <= 0) {
39-
return false;
40-
}
41-
else if (!noncur_expiration.empty() && noncur_expiration.get_days() <=0) {
42-
return false;
43-
}
44-
else if (!mp_expiration.empty() && mp_expiration.get_days() <= 0) {
38+
else if (!expiration.valid() || !noncur_expiration.valid() || !mp_expiration.valid()) {
4539
return false;
4640
}
4741
return true;
@@ -60,13 +54,16 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
6054
if (rule->get_status().compare("Enabled") == 0) {
6155
op.status = true;
6256
}
63-
if (!rule->get_expiration().empty()) {
57+
if (rule->get_expiration().has_days()) {
6458
op.expiration = rule->get_expiration().get_days();
6559
}
66-
if (!rule->get_noncur_expiration().empty()) {
60+
if (rule->get_expiration().has_date()) {
61+
op.expiration_date = ceph::from_iso_8601(rule->get_expiration().get_date());
62+
}
63+
if (rule->get_noncur_expiration().has_days()) {
6764
op.noncur_expiration = rule->get_noncur_expiration().get_days();
6865
}
69-
if (!rule->get_mp_expiration().empty()) {
66+
if (rule->get_mp_expiration().has_days()) {
7067
op.mp_expiration = rule->get_mp_expiration().get_days();
7168
}
7269
op.dm_expiration = rule->get_dm_expiration();
@@ -76,7 +73,7 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
7673

7774
int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule)
7875
{
79-
if (!rule->validate()) {
76+
if (!rule->valid()) {
8077
return -EINVAL;
8178
}
8279
string id;
@@ -92,9 +89,22 @@ int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule)
9289
return 0;
9390
}
9491

92+
bool RGWLifecycleConfiguration::has_same_action(const lc_op& first, const lc_op& second) {
93+
if ((first.expiration > 0 || first.expiration_date != boost::none) &&
94+
(second.expiration > 0 || second.expiration_date != boost::none)) {
95+
return true;
96+
} else if (first.noncur_expiration > 0 && second.noncur_expiration > 0) {
97+
return true;
98+
} else if (first.mp_expiration > 0 && second.mp_expiration > 0) {
99+
return true;
100+
} else {
101+
return false;
102+
}
103+
}
104+
95105
//Rules are conflicted: if one rule's prefix starts with other rule's prefix, and these two rules
96106
//define same action.
97-
bool RGWLifecycleConfiguration::validate()
107+
bool RGWLifecycleConfiguration::valid()
98108
{
99109
if (prefix_map.size() < 2) {
100110
return true;
@@ -107,9 +117,7 @@ bool RGWLifecycleConfiguration::validate()
107117
string c_pre = cur_iter->first;
108118
string n_pre = next_iter->first;
109119
if (n_pre.compare(0, c_pre.length(), c_pre) == 0) {
110-
if ((cur_iter->second.expiration > 0 && next_iter->second.expiration > 0) ||
111-
(cur_iter->second.noncur_expiration > 0 && next_iter->second.noncur_expiration > 0) ||
112-
(cur_iter->second.mp_expiration > 0 && next_iter->second.mp_expiration > 0)) {
120+
if (has_same_action(cur_iter->second, next_iter->second)) {
113121
return false;
114122
} else {
115123
++next_iter;
@@ -346,7 +354,12 @@ int RGWLC::bucket_lc_process(string& shard_id)
346354
list_op.params.list_versions = bucket_info.versioned();
347355
if (!bucket_info.versioned()) {
348356
for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
349-
if (!prefix_iter->second.status || prefix_iter->second.expiration <=0) {
357+
if (!prefix_iter->second.status ||
358+
(prefix_iter->second.expiration <=0 && prefix_iter->second.expiration_date == boost::none)) {
359+
continue;
360+
}
361+
if (prefix_iter->second.expiration_date != boost::none &&
362+
ceph_clock_now() < ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date)) {
350363
continue;
351364
}
352365
list_op.params.prefix = prefix_iter->first;
@@ -361,17 +374,22 @@ int RGWLC::bucket_lc_process(string& shard_id)
361374
ldout(cct, 0) << "ERROR: store->list_objects():" <<dendl;
362375
return ret;
363376
}
364-
377+
365378
utime_t now = ceph_clock_now();
366-
379+
bool is_expired;
367380
for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
368381
rgw_obj_key key(obj_iter->key);
369382

370383
if (!key.ns.empty()) {
371384
continue;
372385
}
373-
374-
if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration)) {
386+
if (prefix_iter->second.expiration_date != boost::none) {
387+
//we have checked it before
388+
is_expired = true;
389+
} else {
390+
is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration);
391+
}
392+
if (is_expired) {
375393
RGWObjectCtx rctx(store);
376394
rgw_obj obj(bucket_info.bucket, key);
377395
RGWObjState *state;
@@ -396,6 +414,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
396414
rgw_obj_key pre_marker;
397415
for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
398416
if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0
417+
&& prefix_iter->second.expiration_date == boost::none
399418
&& prefix_iter->second.noncur_expiration <= 0 && !prefix_iter->second.dm_expiration)) {
400419
continue;
401420
}
@@ -427,10 +446,13 @@ int RGWLC::bucket_lc_process(string& shard_id)
427446
bool remove_indeed = true;
428447
int expiration;
429448
bool skip_expiration;
449+
bool is_expired;
430450
for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
431451
skip_expiration = false;
452+
is_expired = false;
432453
if (obj_iter->is_current()) {
433-
if (prefix_iter->second.expiration <= 0 && !prefix_iter->second.dm_expiration) {
454+
if (prefix_iter->second.expiration <= 0 && prefix_iter->second.expiration_date == boost::none
455+
&& !prefix_iter->second.dm_expiration) {
434456
continue;
435457
}
436458
if (obj_iter->is_delete_marker()) {
@@ -450,8 +472,14 @@ int RGWLC::bucket_lc_process(string& shard_id)
450472
}
451473
mtime = obj_iter->meta.mtime;
452474
expiration = prefix_iter->second.expiration;
453-
if (!skip_expiration && expiration <= 0) {
475+
if (!skip_expiration && expiration <= 0 && prefix_iter->second.expiration_date == boost::none) {
454476
continue;
477+
} else if (!skip_expiration) {
478+
if (expiration > 0) {
479+
is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration);
480+
} else {
481+
is_expired = now >= ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date);
482+
}
455483
}
456484
} else {
457485
if (prefix_iter->second.noncur_expiration <=0) {
@@ -460,8 +488,9 @@ int RGWLC::bucket_lc_process(string& shard_id)
460488
remove_indeed = true;
461489
mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime;
462490
expiration = prefix_iter->second.noncur_expiration;
491+
is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration);
463492
}
464-
if (skip_expiration || obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) {
493+
if (skip_expiration || is_expired) {
465494
if (obj_iter->is_visible()) {
466495
RGWObjectCtx rctx(store);
467496
rgw_obj obj(bucket_info.bucket, obj_iter->key);

‎src/rgw/rgw_lc.h

+36-8
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "include/rados/librados.hpp"
1313
#include "common/Mutex.h"
1414
#include "common/Cond.h"
15+
#include "common/iso_8601.h"
1516
#include "common/Thread.h"
1617
#include "rgw_common.h"
1718
#include "rgw_rados.h"
@@ -38,29 +39,54 @@ class LCExpiration
3839
{
3940
protected:
4041
string days;
42+
//At present only current object has expiration date
43+
string date;
4144
public:
4245
LCExpiration() {}
4346
~LCExpiration() {}
4447

4548
void encode(bufferlist& bl) const {
46-
ENCODE_START(2, 2, bl);
49+
ENCODE_START(3, 2, bl);
4750
::encode(days, bl);
51+
::encode(date, bl);
4852
ENCODE_FINISH(bl);
4953
}
5054
void decode(bufferlist::iterator& bl) {
51-
DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
55+
DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
5256
::decode(days, bl);
57+
if (struct_v >= 3) {
58+
::decode(date, bl);
59+
}
5360
DECODE_FINISH(bl);
5461
}
5562
void dump(Formatter *f) const;
5663
// static void generate_test_instances(list<ACLOwner*>& o);
5764
void set_days(const string& _days) { days = _days; }
58-
string get_days_str() const{
65+
string get_days_str() const {
5966
return days;
6067
}
61-
int get_days() {return atoi(days.c_str()); }
62-
bool empty() const{
63-
return days.empty();
68+
int get_days() const {return atoi(days.c_str()); }
69+
bool has_days() const {
70+
return !days.empty();
71+
}
72+
void set_date(const string& _date) { date = _date; }
73+
string get_date() const {
74+
return date;
75+
}
76+
bool has_date() const {
77+
return !date.empty();
78+
}
79+
bool empty() const {
80+
return days.empty() && date.empty();
81+
}
82+
bool valid() const {
83+
if (!days.empty() && !date.empty()) {
84+
return false;
85+
} else if (!days.empty() && get_days() <= 0) {
86+
return false;
87+
}
88+
//We've checked date in xml parsing
89+
return true;
6490
}
6591
};
6692
WRITE_CLASS_ENCODER(LCExpiration)
@@ -138,7 +164,7 @@ class LCRule
138164
dm_expiration = _dm_expiration;
139165
}
140166

141-
bool validate();
167+
bool valid();
142168

143169
void encode(bufferlist& bl) const {
144170
ENCODE_START(4, 1, bl);
@@ -179,6 +205,7 @@ struct lc_op
179205
int expiration;
180206
int noncur_expiration;
181207
int mp_expiration;
208+
boost::optional<ceph::real_time> expiration_date;
182209

183210
lc_op() : status(false), dm_expiration(false), expiration(0), noncur_expiration(0), mp_expiration(0) {}
184211

@@ -191,6 +218,7 @@ class RGWLifecycleConfiguration
191218
map<string, lc_op> prefix_map;
192219
multimap<string, LCRule> rule_map;
193220
bool _add_rule(LCRule *rule);
221+
bool has_same_action(const lc_op& first, const lc_op& second);
194222
public:
195223
RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
196224
RGWLifecycleConfiguration() : cct(NULL) {}
@@ -225,7 +253,7 @@ class RGWLifecycleConfiguration
225253

226254
int check_and_add_rule(LCRule* rule);
227255

228-
bool validate();
256+
bool valid();
229257

230258
multimap<string, LCRule>& get_rule_map() { return rule_map; }
231259
map<string, lc_op>& get_prefix_map() { return prefix_map; }

‎src/rgw/rgw_lc_s3.cc

+16-4
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ using namespace std;
1616
bool LCExpiration_S3::xml_end(const char * el) {
1717
LCDays_S3 *lc_days = static_cast<LCDays_S3 *>(find_first("Days"));
1818
LCDeleteMarker_S3 *lc_dm = static_cast<LCDeleteMarker_S3 *>(find_first("ExpiredObjectDeleteMarker"));
19+
LCDate_S3 *lc_date = static_cast<LCDate_S3 *>(find_first("Date"));
1920

20-
if ((!lc_days && !lc_dm) || (lc_days && lc_dm)) {
21+
if ((!lc_days && !lc_dm && !lc_date) || (lc_days && lc_dm)
22+
|| (lc_days && lc_date) || (lc_dm && lc_date)) {
2123
return false;
2224
}
2325
if (lc_days) {
@@ -27,6 +29,12 @@ bool LCExpiration_S3::xml_end(const char * el) {
2729
if (!dm_expiration) {
2830
return false;
2931
}
32+
} else {
33+
date = lc_date->get_data();
34+
//We need return xml error according to S3
35+
if (boost::none == ceph::from_iso_8601(date)) {
36+
return false;
37+
}
3038
}
3139
return true;
3240
}
@@ -96,8 +104,10 @@ bool LCRule_S3::xml_end(const char *el) {
96104
return false;
97105
} else {
98106
if (lc_expiration) {
99-
if (!lc_expiration->empty()) {
107+
if (lc_expiration->has_days()) {
100108
expiration.set_days(lc_expiration->get_days_str());
109+
} else if (lc_expiration->has_date()) {
110+
expiration.set_date(lc_expiration->get_date());
101111
} else {
102112
dm_expiration = lc_expiration->get_dm_expiration();
103113
}
@@ -119,7 +129,7 @@ void LCRule_S3::to_xml(CephContext *cct, ostream& out) {
119129
out << "<Prefix>" << prefix << "</Prefix>";
120130
out << "<Status>" << status << "</Status>";
121131
if (!expiration.empty() || dm_expiration) {
122-
LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
132+
LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
123133
expir.to_xml(out);
124134
}
125135
if (!noncur_expiration.empty()) {
@@ -143,7 +153,7 @@ int RGWLifecycleConfiguration_S3::rebuild(RGWRados *store, RGWLifecycleConfigura
143153
if (ret < 0)
144154
return ret;
145155
}
146-
if (!dest.validate()) {
156+
if (!dest.valid()) {
147157
ret = -ERR_INVALID_REQUEST;
148158
}
149159
return ret;
@@ -178,6 +188,8 @@ XMLObj *RGWLCXMLParser_S3::alloc_obj(const char *el)
178188
obj = new LCExpiration_S3();
179189
} else if (strcmp(el, "Days") == 0) {
180190
obj = new LCDays_S3();
191+
} else if (strcmp(el, "Date") == 0) {
192+
obj = new LCDate_S3();
181193
} else if (strcmp(el, "ExpiredObjectDeleteMarker") == 0) {
182194
obj = new LCDeleteMarker_S3();
183195
} else if (strcmp(el, "NoncurrentVersionExpiration") == 0) {

0 commit comments

Comments
 (0)