diff --git a/core/base/apiModule.go b/core/base/apiModule.go index 37371d7..3d3a945 100644 --- a/core/base/apiModule.go +++ b/core/base/apiModule.go @@ -424,13 +424,13 @@ func ListObjectsWithDestinationPolicyUpdatedSince(orgID string, since int64) ([] } // ListObjectsWithFilters provides a list of objects that satisfy the given conditions -func ListObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) { +func ListObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) { apiLock.RLock() defer apiLock.RUnlock() common.HealthStatus.ClientRequestReceived() - objects, err := store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore) + objects, err := store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore, deleted) if trace.IsLogging(logger.DEBUG) { trace.Debug("In ListObjectsWithFilters. Get %s. Returned %d objects\n", orgID, len(objects)) diff --git a/core/base/apiServer.go b/core/base/apiServer.go index 78f2904..01a9f85 100644 --- a/core/base/apiServer.go +++ b/core/base/apiServer.go @@ -857,14 +857,19 @@ func handleObjectRequest(orgID string, objectType string, objectID string, write // type: string // - name: noData // in: query -// description: Fetch the objects with noData marked to true +// description: Specify true or false. Fetch the objects with noData marked to true/false. If not specified, object will not be filtered by "noData" field // required: false -// type: boolean +// type: string // - name: expirationTimeBefore // in: query // description: Fetch the objects with expiration time before specified timestamp in RFC3339 format // required: false // type: string +// - name: deleted +// in: query +// description: Specify true or false. Fetch the object with deleted marked to true/false. If not specified, object will not be filtered by "deleted" field +// required: false +// type: string // // responses: // '200': @@ -993,14 +998,25 @@ func handleListObjectsWithFilters(orgID string, writer http.ResponseWriter, requ } } + var deleted *bool + deletedString := request.URL.Query().Get("deleted") + if deletedString != "" { + deletedValue, err := strconv.ParseBool(deletedString) + deleted = &deletedValue + if err != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + } + var objects []common.MetaData var err error if trace.IsLogging(logger.DEBUG) { - trace.Debug("In handleListObjectsWithFilters, get objects with %s %s %s %s %s %d %s %s %s %s %s %s\n", orgID, destinationPolicyString, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noDataString, expirationTimeBeforeString) + trace.Debug("In handleListObjectsWithFilters, get objects with %s %s %s %s %s %d %s %s %s %s %s %s %s\n", orgID, destinationPolicyString, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noDataString, expirationTimeBeforeString, deletedString) } - if objects, err = ListObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBeforeString); err != nil { + if objects, err = ListObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBeforeString, deleted); err != nil { communications.SendErrorResponse(writer, err, "Failed to fetch the list of objects with given conditions. Error: ", 0) } else { if len(objects) == 0 { diff --git a/core/storage/boltStorage.go b/core/storage/boltStorage.go index a3646db..b38d4ac 100644 --- a/core/storage/boltStorage.go +++ b/core/storage/boltStorage.go @@ -545,7 +545,7 @@ func (store *BoltStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID } // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions -func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) { +func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) { result := make([]common.MetaData, 0) function := func(object boltObject) { if orgID == object.Meta.DestOrgID { @@ -656,6 +656,10 @@ func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPo } } + if deleted != nil && *deleted != object.Meta.Deleted { + return + } + // if expirationTimeBefore is not satisfy, return result = append(result, object.Meta) diff --git a/core/storage/cache.go b/core/storage/cache.go index ff6d27c..cba5dc6 100644 --- a/core/storage/cache.go +++ b/core/storage/cache.go @@ -151,8 +151,8 @@ func (store *Cache) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID strin } // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions -func (store *Cache) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) { - return store.Store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore) +func (store *Cache) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) { + return store.Store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore, deleted) } // RetrieveAllObjects returns the list of all the objects of the specified type diff --git a/core/storage/inMemoryStorage.go b/core/storage/inMemoryStorage.go index af6383f..3db6f10 100644 --- a/core/storage/inMemoryStorage.go +++ b/core/storage/inMemoryStorage.go @@ -396,7 +396,7 @@ func (store *InMemoryStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(o } // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions -func (store *InMemoryStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) { +func (store *InMemoryStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) { return nil, nil } diff --git a/core/storage/mongoStorage.go b/core/storage/mongoStorage.go index f6215f4..a7f12df 100644 --- a/core/storage/mongoStorage.go +++ b/core/storage/mongoStorage.go @@ -766,7 +766,7 @@ func (store *MongoStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgI } // RetrieveObjectsWithFilters returns the list of all the objects that meet the given conditions -func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) { +func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) { result := []object{} query := bson.M{ @@ -832,6 +832,10 @@ func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationP query["metadata.expiration"] = subquery } + if deleted != nil { + query["metadata.deleted"] = *deleted + } + if err := store.fetchAll(objects, query, nil, &result); err != nil { switch err { case mgo.ErrNotFound: diff --git a/core/storage/storage.go b/core/storage/storage.go index 8328872..58ee4c8 100644 --- a/core/storage/storage.go +++ b/core/storage/storage.go @@ -89,7 +89,7 @@ type Storage interface { RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID string, since int64) ([]common.ObjectDestinationPolicy, common.SyncServiceError) // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions - RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) + RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) // RetrieveAllObjects returns the list of all the objects of the specified type RetrieveAllObjects(orgID string, objectType string) ([]common.ObjectDestinationPolicy, common.SyncServiceError) diff --git a/core/storage/storage_test.go b/core/storage/storage_test.go index b5e744e..170dad7 100644 --- a/core/storage/storage_test.go +++ b/core/storage/storage_test.go @@ -513,9 +513,9 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { }, Expiration: "2014-08-14T14:00:00Z", }}, - {common.MetaData{ObjectID: "5a", ObjectType: "type2", DestOrgID: "myorg111", DestType: "myDestType5", DestID: "myDestID5a", NoData: true, Expiration: ""}}, - {common.MetaData{ObjectID: "5b", ObjectType: "type2", DestOrgID: "myorg111", DestType: "myDestType5", DestID: "myDestID5b", NoData: false, Expiration: ""}}, - {common.MetaData{ObjectID: "5c", ObjectType: "type2", DestOrgID: "myorg111", DestinationsList: destArray, NoData: false, Expiration: ""}}, + {common.MetaData{ObjectID: "5a", ObjectType: "type2", DestOrgID: "myorg111", DestType: "myDestType5", DestID: "myDestID5a", NoData: true, Expiration: "", Deleted: true}}, + {common.MetaData{ObjectID: "5b", ObjectType: "type2", DestOrgID: "myorg111", DestType: "myDestType5", DestID: "myDestID5b", NoData: false, Expiration: "", Deleted: true}}, + {common.MetaData{ObjectID: "5c", ObjectType: "type2", DestOrgID: "myorg111", DestinationsList: destArray, NoData: false, Expiration: "", Deleted: true}}, } for _, test := range tests { @@ -535,7 +535,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } } - objects, err := store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", nil, "") + objects, err := store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with filters. Error: %s\n", err) } @@ -545,7 +545,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { destinationPolicy := true expectedResultCount := 5 - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -554,7 +554,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 2 - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "plover", "xyzzy", "", 0, "", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "plover", "xyzzy", "", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -563,7 +563,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 1 - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "plover", "xyzzy", "d", 0, "", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "plover", "xyzzy", "d", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -572,7 +572,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 1 - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "d", 0, "", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "d", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -581,7 +581,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 1 - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "type2", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "type2", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -591,7 +591,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { destinationPolicy = false expectedResultCount = 6 - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -599,7 +599,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { t.Errorf("Retrieved %d objects without destination policy. Expected %d\n", len(objects), expectedResultCount) } - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "d", 0, "", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "d", 0, "", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -608,7 +608,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 6 - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "myDestType5", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "myDestType5", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -617,7 +617,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 2 - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "myDestType5", "myDestID5a", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "myDestType5", "myDestID5a", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -626,7 +626,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 2 - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "myDestType5", "myDestID5c", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "myDestType5", "myDestID5c", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -635,7 +635,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 4 - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "type2", "", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "type2", "", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -644,7 +644,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 1 - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "type2", "5a", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "type2", "5a", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -652,7 +652,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { t.Errorf("Retrieved %d objects with given destination type. Expected %d\n", len(objects), expectedResultCount) } - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "5a", "", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "5a", "", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -661,7 +661,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { } expectedResultCount = 3 - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "type2", "", "myDestType5", "", nil, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "type2", "", "myDestType5", "", nil, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -671,7 +671,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { expectedResultCount = 7 noData := true - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", &noData, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", &noData, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -681,7 +681,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { expectedResultCount = 2 destinationPolicy = false - objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "", "", "", "", &noData, "") + objects, err = store.RetrieveObjectsWithFilters("myorg111", &destinationPolicy, "", "", "", 0, "", "", "", "", &noData, "", nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -691,7 +691,7 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { expectedResultCount = 2 expirationTimeBefore := "2013-08-15T14:00:00Z" - objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", nil, expirationTimeBefore) + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", nil, expirationTimeBefore, nil) if err != nil { t.Errorf("Failed to retrieve the objects with a destination policy. Error: %s\n", err) } @@ -699,6 +699,26 @@ func testGetObjectWithFilters(storageType string, t *testing.T) { t.Errorf("Retrieved %d objects with given destination type. Expected %d\n", len(objects), expectedResultCount) } + expectedResultCount = 3 + deleted := true + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", nil, "", &deleted) + if err != nil { + t.Errorf("Failed to retrieve the objects with deleted = true. Error: %s\n", err) + } + if len(objects) != expectedResultCount { + t.Errorf("(%s) Retrieved %d objects with deleted set to true. Expected %d\n", storageType, len(objects), expectedResultCount) + } + + expectedResultCount = len(tests) - expectedResultCount + deleted = false + objects, err = store.RetrieveObjectsWithFilters("myorg111", nil, "", "", "", 0, "", "", "", "", nil, "", &deleted) + if err != nil { + t.Errorf("Failed to retrieve the objects with deleted = true. Error: %s\n", err) + } + if len(objects) != expectedResultCount { + t.Errorf("(%s) Retrieved %d objects with deleted set to true. Expected %d\n", storageType, len(objects), expectedResultCount) + } + } func testStorageObjectActivation(storageType string, t *testing.T) { diff --git a/swagger.json b/swagger.json index 1b9795f..d47184a 100644 --- a/swagger.json +++ b/swagger.json @@ -1824,8 +1824,8 @@ "in": "query" }, { - "type": "boolean", - "description": "Fetch the objects with noData marked to true", + "type": "string", + "description": "Specify true or false. Fetch the objects with noData marked to true/false. If not specified, object will not be filtered by \"noData\" field", "name": "noData", "in": "query" }, @@ -1834,6 +1834,12 @@ "description": "Fetch the objects with expiration time before specified timestamp in RFC3339 format", "name": "expirationTimeBefore", "in": "query" + }, + { + "type": "string", + "description": "Specify true or false. Fetch the object with deleted marked to true/false. If not specified, object will not be filtered by \"deleted\" field", + "name": "deleted", + "in": "query" } ], "responses": {