Skip to content

Commit

Permalink
Issue 1115 - add objectType/ID filters
Browse files Browse the repository at this point in the history
  • Loading branch information
@LiilyZhang
LiilyZhang committed Aug 20, 2019
1 parent ff53afe commit 0ce3081
Show file tree
Hide file tree
Showing 10 changed files with 256 additions and 67 deletions.
6 changes: 3 additions & 3 deletions core/base/apiModule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,14 +345,14 @@ func ListObjectsWithDestinationPolicyUpdatedSince(orgID string, since int64) ([]
return objects, err
}

// GetObjectsByFilter provides a list of objects that satisfy the given conditions
func ListObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
// ListObjectsWithFilters provides a list of objects that satisfy the given conditions
func ListObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
apiLock.RLock()
defer apiLock.RUnlock()

common.HealthStatus.ClientRequestReceived()

objects, err := store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, destinationType, destinationID, noData, expirationTimeBefore)
objects, err := store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore)

if trace.IsLogging(logger.DEBUG) {
trace.Debug("In ListObjectsWithFilters. Get %s. Returned %d objects\n", orgID, len(objects))
Expand Down
21 changes: 19 additions & 2 deletions core/base/apiServer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,16 @@ func handleObjectRequest(orgID string, objectType string, objectID string, write
// description: Objects that have a Destination Policy which was updated since the specified timestamp in RFC3339 should be fetched.
// required: false
// type: string
// - name: objectType
// in: query
// description: Fetch the objects with given object type
// required: false
// type: string
// - name: objectID
// in: query
// description: Fetch the objects with given object id
// required: false
// type: string
// - name: destinationType
// in: query
// description: Fetch the objects with given destination type
Expand Down Expand Up @@ -741,6 +751,13 @@ func handleListObjectsWithFilters(orgID string, writer http.ResponseWriter, requ
}
}

objectType := request.URL.Query().Get("objectType")
objectID := ""

if objectType != "" {
objectID = request.URL.Query().Get("objectID")
}

destinationType := request.URL.Query().Get("destinationType")
destinationID := ""
if destinationType != "" {
Expand Down Expand Up @@ -772,10 +789,10 @@ func handleListObjectsWithFilters(orgID string, writer http.ResponseWriter, requ
var err error

if trace.IsLogging(logger.DEBUG) {
trace.Debug("In handleListObjectsWithFilters, get objects with %s %s %s %s %s %d %s %s %s %s\n", orgID, destinationPolicyString, dpServiceOrgID, dpServiceName, dpPropertyName, since, destinationType, destinationID, noDataString, expirationTimeBeforeString)
trace.Debug("In handleListObjectsWithFilters, get objects with %s %s %s %s %s %d %s %s %s %s %s %s\n", orgID, destinationPolicyString, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noDataString, expirationTimeBeforeString)
}

if objects, err = ListObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, destinationType, destinationID, noData, expirationTimeBeforeString); err != nil {
if objects, err = ListObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBeforeString); err != nil {
communications.SendErrorResponse(writer, err, "Failed to fetch the list of objects with given conditions. Error: ", 0)
} else {
if len(objects) == 0 {
Expand Down
92 changes: 61 additions & 31 deletions core/base/apiServer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -751,13 +751,26 @@ func testGetObjectsWithFiltersHelper(storageProvider string, t *testing.T) {
t.Errorf("StoreObject failed: %s", err.Error())
}

destinations := []common.Destination{
common.Destination{DestOrgID: "myorgObjFilter", DestType: "myDestType5", DestID: "myDestID5a", Communication: common.MQTTProtocol},
common.Destination{DestOrgID: "myorgObjFilter", DestType: "myDestType5", DestID: "myDestID5a", Communication: common.MQTTProtocol},
}

for _, destination := range destinations {
if err := store.StoreDestination(destination); err != nil {
t.Errorf("Failed to store detination %#v. Error: %s\n", destination, err)
}
}

tests := []struct {
method string
orgID string
destinationPolicy string
destService string
destPropname string
since string
objType string
objID string
destType string
destID string
noData string
Expand All @@ -767,40 +780,38 @@ func testGetObjectsWithFiltersHelper(storageProvider string, t *testing.T) {
testID int
}{
// Must be first test
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "", "", http.StatusOK, 7, 0},
{http.MethodGet, "myorgObjFilter", "true", "", "", "", "", "", "", "", http.StatusOK, 5, 1},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "", "", http.StatusOK, 2, 2},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "", "", "", "", http.StatusOK, 12, 0},
{http.MethodGet, "myorgObjFilter", "true", "", "", "", "", "", "", "", "", "", http.StatusOK, 6, 1},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "", "", "", "", http.StatusOK, 6, 2},
// Must be second test
{http.MethodGet, "myorgObjFilter", "true", "plover/testerService1", "", "", "", "", "", "", http.StatusOK, 2, 3},
{http.MethodGet, "myorgObjFilter", "true", "plover/testerService1", "b", "", "", "", "", "", http.StatusOK, 1, 4},
{http.MethodGet, "myorgObjFilter", "", "plover/testerService1", "b", "", "", "", "", "", http.StatusOK, 7, 5},
{http.MethodGet, "myorgObjFilter", "true", "", "b", "", "", "", "", "", http.StatusOK, 4, 6},
{http.MethodGet, "myorgObjFilter", "true", "", "", "2000-08-14T14:00:00Z", "", "", "", "", http.StatusOK, 5, 7},
{http.MethodGet, "myorgObjFilter", "true", "", "", "2030-08-14T14:00:00Z", "", "", "", "", http.StatusNotFound, 0, 8},
{http.MethodGet, "myorgObjFilter", "true", "", "", "", "", "", "false", "", http.StatusOK, 1, 9},
{http.MethodGet, "myorgObjFilter", "true", "", "", "", "", "", "true", "", http.StatusOK, 4, 10},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "false", "", http.StatusOK, 1, 11},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "true", "", http.StatusOK, 1, 12},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "false", "", http.StatusOK, 2, 13},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "true", "", http.StatusOK, 5, 14},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "myDestType5", "", "", "", http.StatusOK, 2, 15},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "myDestType5", "myDestID5a", "", "", http.StatusOK, 1, 16},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "", "2012-08-15T14:00:00Z", http.StatusOK, 2, 17},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "", "2012-08-15T14:00:00Z", http.StatusNotFound, 0, 18},
{http.MethodGet, "myorgObjFilter1", "true", "", "", "", "", "", "", "", http.StatusOK, 1, 19},
{http.MethodGet, "myorgObjFilter", "aaa", "", "", "", "", "", "", "", http.StatusBadRequest, 0, 20},
{http.MethodPut, "myorgObjFilter", "true", "", "", "", "", "", "", "", http.StatusMethodNotAllowed, 0, 21},
{http.MethodGet, "myorgObjFilter2", "true", "", "", "", "", "", "", "", http.StatusNotFound, 0, 22},
}

//sinceFormat := time.Unix(since, 0).Format(time.RFC3339)
//tests[5].since = sinceFormat
//tests[1].expectedCount = totalCount - 1
{http.MethodGet, "myorgObjFilter", "true", "plover/testerService1", "", "", "", "", "", "", "", "", http.StatusOK, 2, 3},
{http.MethodGet, "myorgObjFilter", "true", "plover/testerService1", "b", "", "", "", "", "", "", "", http.StatusOK, 1, 4},
{http.MethodGet, "myorgObjFilter", "", "plover/testerService1", "b", "", "", "", "", "", "", "", http.StatusOK, 12, 5},
{http.MethodGet, "myorgObjFilter", "true", "", "b", "", "", "", "", "", "", "", http.StatusOK, 5, 6},
{http.MethodGet, "myorgObjFilter", "true", "", "", "2000-08-14T14:00:00Z", "", "", "", "", "", "", http.StatusOK, 6, 7},
{http.MethodGet, "myorgObjFilter", "true", "", "", "2030-08-14T14:00:00Z", "", "", "", "", "", "", http.StatusNotFound, 0, 8},
{http.MethodGet, "myorgObjFilter", "true", "", "", "", "", "", "", "", "false", "", http.StatusOK, 2, 9},
{http.MethodGet, "myorgObjFilter", "true", "", "", "", "", "", "", "", "true", "", http.StatusOK, 4, 10},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "", "", "false", "", http.StatusOK, 4, 11},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "", "", "true", "", http.StatusOK, 2, 12},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "", "", "false", "", http.StatusOK, 6, 13},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "", "", "true", "", http.StatusOK, 6, 14},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "myDestType5", "", "", "", http.StatusOK, 6, 15},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "myDestType5", "myDestID5a", "", "", http.StatusOK, 4, 16},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "type2", "", "myDestType5", "", "", "", http.StatusOK, 3, 17},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "type2", "", "myDestType5", "myDestID5a", "", "", http.StatusOK, 2, 18},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "type2", "7c", "myDestType5", "myDestID5a", "", "", http.StatusOK, 1, 19},
{http.MethodGet, "myorgObjFilter", "", "", "", "", "", "", "", "", "", "2012-08-15T14:00:00Z", http.StatusOK, 2, 20},
{http.MethodGet, "myorgObjFilter", "false", "", "", "", "", "", "", "", "", "2012-08-15T14:00:00Z", http.StatusNotFound, 0, 21},
{http.MethodGet, "myorgObjFilter1", "true", "", "", "", "", "", "", "", "", "", http.StatusOK, 1, 22},
{http.MethodGet, "myorgObjFilter", "aaa", "", "", "", "", "", "", "", "", "", http.StatusBadRequest, 0, 23},
{http.MethodPut, "myorgObjFilter", "true", "", "", "", "", "", "", "", "", "", http.StatusMethodNotAllowed, 0, 24},
{http.MethodGet, "myorgObjFilter2", "true", "", "", "", "", "", "", "", "", "", http.StatusNotFound, 0, 25},
}

for _, test := range tests {
//urlString := fmt.Sprintf("%s?destination_policy=true&since=%d", test.orgID, test.since)
urlString := fmt.Sprintf("%s?filters=true&destinationPolicy=%s&dpPropertyName=%s&dpService=%s&since=%s&destinationType=%s&destinationID=%s&noData=%s&expirationTimeBefore=%s",
test.orgID, test.destinationPolicy, test.destPropname, test.destService, test.since, test.destType, test.destID, test.noData, test.expirationBefore)
urlString := fmt.Sprintf("%s?filters=true&destinationPolicy=%s&dpPropertyName=%s&dpService=%s&since=%s&objectType=%s&objectID=%s&destinationType=%s&destinationID=%s&noData=%s&expirationTimeBefore=%s",
test.orgID, test.destinationPolicy, test.destPropname, test.destService, test.since, test.objType, test.objID, test.destType, test.destID, test.noData, test.expirationBefore)
writer := newAPIServerTestResponseWriter()
request, _ := http.NewRequest(test.method, urlString, nil)
request.SetBasicAuth("testerAdmin@"+test.orgID, "")
Expand All @@ -826,6 +837,7 @@ func testGetObjectsWithFiltersHelper(storageProvider string, t *testing.T) {
}

func loadTestMetaData(nodeType string, orgID string) (int64, int, error) {
destArray := []string{"myDestType5:myDestID5a", "myDestType5:myDestID5c"}
testData := []common.MetaData{
common.MetaData{ObjectID: "1", ObjectType: "type1", DestOrgID: "myorgObjFilter1", NoData: true,
DestinationPolicy: &common.Policy{
Expand Down Expand Up @@ -913,6 +925,24 @@ func loadTestMetaData(nodeType string, orgID string) (int64, int, error) {
},
common.MetaData{ObjectID: "5a", ObjectType: "type1", DestOrgID: "myorgObjFilter", DestType: "myDestType5", DestID: "myDestID5a", NoData: true, Expiration: "2015-08-14T14:00:00Z"},
common.MetaData{ObjectID: "5b", ObjectType: "type1", DestOrgID: "myorgObjFilter", DestType: "myDestType5", DestID: "myDestID5b", NoData: false, Expiration: "2015-08-14T14:00:00Z"},
common.MetaData{ObjectID: "5c", ObjectType: "type1", DestOrgID: "myorgObjFilter", DestinationsList: destArray, NoData: false, Expiration: "2015-08-14T14:00:00Z"},
common.MetaData{ObjectID: "6", ObjectType: "type2", DestOrgID: "myorgObjFilter", NoData: false,
DestinationPolicy: &common.Policy{
Properties: []common.PolicyProperty{
{Name: "a", Value: float64(1)},
{Name: "b", Value: "zxcv"},
{Name: "c", Value: true, Type: "bool"},
},
Constraints: []string{"Plover=34", "asdf=true"},
Services: []common.ServiceID{
{OrgID: "plover", Arch: "amd64", ServiceName: "testerService4", Version: "0.0.1"},
},
},
Expiration: "2014-08-14T14:00:00Z",
},
common.MetaData{ObjectID: "7a", ObjectType: "type2", DestOrgID: "myorgObjFilter", DestType: "myDestType5", DestID: "myDestID5a", NoData: true, Expiration: "2015-08-14T14:00:00Z"},
common.MetaData{ObjectID: "7b", ObjectType: "type2", DestOrgID: "myorgObjFilter", DestType: "myDestType5", DestID: "myDestID5b", NoData: false, Expiration: "2015-08-14T14:00:00Z"},
common.MetaData{ObjectID: "7c", ObjectType: "type2", DestOrgID: "myorgObjFilter", DestinationsList: destArray, NoData: false, Expiration: "2015-08-14T14:00:00Z"},
}

var since int64
Expand Down
43 changes: 39 additions & 4 deletions core/storage/boltStorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ func (store *BoltStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID
}

// RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
result := make([]common.MetaData, 0)
function := func(object boltObject) {
if orgID == object.Meta.DestOrgID {
Expand Down Expand Up @@ -529,17 +529,52 @@ func (store *BoltStorage) RetrieveObjectsWithFilters(orgID string, destinationPo
}
}

// check objectType and objectID
if objectType != "" {
if objectType != object.Meta.ObjectType {
return
}
if objectID != "" {
if objectID != object.Meta.ObjectID {
return
}
}
}

// check destinationType and destinationID
if destinationType != "" {
if destinationType != object.Meta.DestType {
return
} else {
if object.Meta.DestType != "" {
if destinationType != object.Meta.DestType {
return
}
if destinationID != "" {
if destinationID != object.Meta.DestID {
return
}
}

} else { // check object.Meta.DestinationsList (destinationType: destinationID)
checkedDestList := false
for _, dest := range object.Meta.DestinationsList {
if destinationID != "" {
if dest == destinationType+":"+destinationID {
checkedDestList = true
break
}
} else {
parts := strings.SplitN(dest, ":", 2)
if len(parts) == 2 && parts[0] == destinationType {
checkedDestList = true
break
}

}
}
if !checkedDestList {
return
}
}

}

if noData != nil {
Expand Down
4 changes: 2 additions & 2 deletions core/storage/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ func (store *Cache) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgID strin
}

// RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
func (store *Cache) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
return store.Store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, destinationType, destinationID, noData, expirationTimeBefore)
func (store *Cache) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
return store.Store.RetrieveObjectsWithFilters(orgID, destinationPolicy, dpServiceOrgID, dpServiceName, dpPropertyName, since, objectType, objectID, destinationType, destinationID, noData, expirationTimeBefore)
}

// RetrieveAllObjects returns the list of all the objects of the specified type
Expand Down
2 changes: 1 addition & 1 deletion core/storage/inMemoryStorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ func (store *InMemoryStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(o
}

// RetrieveObjectsWithFilters returns the list of all othe objects that meet the given conditions
func (store *InMemoryStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
func (store *InMemoryStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
return nil, nil
}

Expand Down
25 changes: 21 additions & 4 deletions core/storage/mongoStorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ func (store *MongoStorage) RetrieveObjectsWithDestinationPolicyUpdatedSince(orgI
}

// RetrieveObjectsWithFilters returns the list of all the objects that meet the given conditions
func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationPolicy *bool, dpServiceOrgID string, dpServiceName string, dpPropertyName string, since int64, objectType string, objectID string, destinationType string, destinationID string, noData *bool, expirationTimeBefore string) ([]common.MetaData, common.SyncServiceError) {
result := []object{}

query := bson.M{
Expand Down Expand Up @@ -718,11 +718,28 @@ func (store *MongoStorage) RetrieveObjectsWithFilters(orgID string, destinationP

}

if objectType != "" {
query["metadata.object-type"] = objectType
if objectID != "" {
query["metadata.object-id"] = objectID
}
}

if destinationType != "" {
query["metadata.destination-type"] = destinationType
if destinationID != "" {
query["metadata.destination-id"] = destinationID
var subquery []bson.M
if destinationID == "" {
subquery = []bson.M{
bson.M{"metadata.destination-type": destinationType},
bson.M{"metadata.destinations-list": bson.M{"$regex": destinationType + ":*"}},
}
} else {
subquery = []bson.M{
bson.M{"metadata.destination-type": destinationType, "metadata.destination-id": destinationID},
bson.M{"metadata.destinations-list": destinationType + ":" + destinationID},
}
}
query["$or"] = subquery

}

if noData != nil {
Expand Down
Loading

0 comments on commit 0ce3081

Please sign in to comment.