Merge pull request #82 from LiilyZhang/issue81

Issue 81 - Deleting an object closely followed by publishing an objec…

core/base/apiModule.go

@@ -424,13 +424,13 @@ func ListObjectsWithDestinationPolicyUpdatedSince(orgID string, since int64) ([]
 }
 
 // ListObjectsWithFilters provides a list of objects that satisfy the given conditions
-func ListObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
+func ListObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) {
 	apiLock.RLock()
 	defer apiLock.RUnlock()
 
 	common.HealthStatus.ClientRequestReceived()
 
-	objects, err := store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore)
+	objects, err := store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore, deleted)
 
 	if trace.IsLogging(logger.DEBUG) {
 		trace.Debug("In ListObjectsWithFilters. Get %s. Returned %d objects\n", orgID, len(objects))

core/base/apiServer.go

@@ -857,14 +857,19 @@ func handleObjectRequest(orgID string, objectType string, objectID string, write
 //   type: string
 // - name: noData
 //   in: query
-//   description: Fetch the objects with noData marked to true
+//   description: Specify true or false. Fetch the objects with noData marked to true/false. If not specified, object will not be filtered by "noData" field
 //   required: false
-//   type: boolean
+//   type: string
 // - name: expirationTimeBefore
 //   in: query
 //   description: Fetch the objects with expiration time before specified timestamp in RFC3339 format
 //   required: false
 //   type: string
+// - name: deleted
+//   in: query
+//   description: Specify true or false. Fetch the object with deleted marked to true/false. If not specified, object will not be filtered by "deleted" field
+//   required: false
+//   type: string
 //
 // responses:
 //   '200':
@@ -993,14 +998,25 @@ func handleListObjectsWithFilters(orgID string, writer http.ResponseWriter, requ
 		}
 	}
 
+	var deleted *bool
+	deletedString := request.URL.Query().Get("deleted")
+	if deletedString != "" {
+		deletedValue, err := strconv.ParseBool(deletedString)
+		deleted = &deletedValue
+		if err != nil {
+			writer.WriteHeader(http.StatusBadRequest)
+			return
+		}
+	}
+
 	var objects []common.MetaData
 	var err error
 
 	if trace.IsLogging(logger.DEBUG) {
-		trace.Debug("In handleListObjectsWithFilters, get objects with %s %s %s %s %s %d %s %s %s %s %s %s\n", orgID, destinationPolicyString, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noDataString, expirationTimeBeforeString)
+		trace.Debug("In handleListObjectsWithFilters, get objects with %s %s %s %s %s %d %s %s %s %s %s %s %s\n", orgID, destinationPolicyString, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noDataString, expirationTimeBeforeString, deletedString)
 	}
 
-	if objects, err = ListObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBeforeString); err != nil {
+	if objects, err = ListObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBeforeString, deleted); err != nil {
 		communications.SendErrorResponse(writer, err, "Failed to fetch the list of objects with given conditions. Error: ", 0)
 	} else {
 		if len(objects) == 0 {

core/storage/boltStorage.go

@@ -545,7 +545,7 @@ func (store *BoltStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID
 }
 
 // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
-func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
+func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) {
 	result := make([]common.MetaData, 0)
 	function := func(object boltObject) {
 		if orgID == object.Meta.DestOrgID {
@@ -656,6 +656,10 @@ func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPo
 				}
 			}
 
+			if deleted != nil && *deleted != object.Meta.Deleted {
+				return
+			}
+
 			// if expirationTimeBefore is not satisfy, return
 			result = append(result, object.Meta)
 

core/storage/cache.go

@@ -151,8 +151,8 @@ func (store *Cache) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID strin
 }
 
 // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
-func (store *Cache) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
-	return store.Store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore)
+func (store *Cache) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) {
+	return store.Store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore, deleted)
 }
 
 // RetrieveAllObjects returns the list of all the objects of the specified type

core/storage/inMemoryStorage.go

@@ -396,7 +396,7 @@ func (store *InMemoryStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(o
 }
 
 // RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
-func (store *InMemoryStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
+func (store *InMemoryStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) {
 	return nil, nil
 }
 

core/storage/mongoStorage.go

@@ -766,7 +766,7 @@ func (store *MongoStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgI
 }
 
 // RetrieveObjectsWithFilters returns the list of all the objects that meet the given conditions
-func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
+func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError) {
 	result := []object{}
 
 	query := bson.M{
@@ -832,6 +832,10 @@ func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationP
 		query["metadata.expiration"] = subquery
 	}
 
+	if deleted != nil {
+		query["metadata.deleted"] = *deleted
+	}
+
 	if err := store.fetchAll(objects, query, nil, &result); err != nil {
 		switch err {
 		case mgo.ErrNotFound:

core/storage/storage.go

@@ -89,7 +89,7 @@ type Storage interface {
 	RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID string, since int64) ([]common.ObjectDestinationPolicy, common.SyncServiceError)
 
 	// RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
-	RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError)
+	RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string, deleted *bool) ([]common.MetaData, common.SyncServiceError)
 
 	// RetrieveAllObjects returns the list of all the objects of the specified type
 	RetrieveAllObjects(orgID string, objectType string) ([]common.ObjectDestinationPolicy, common.SyncServiceError)