Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Add excludeFromIndexes in the proper places for large properties of nested fields #1266

Merged
merged 8 commits into from
Aug 28, 2024
3 changes: 3 additions & 0 deletions protos/protos.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 10 additions & 2 deletions src/entity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,6 @@ export namespace entity {
*/
export function entityToEntityProto(entityObject: EntityObject): EntityProto {
const properties = entityObject.data;
const excludeFromIndexes = entityObject.excludeFromIndexes;

const entityProto: EntityProto = {
key: null,
Expand All @@ -793,6 +792,15 @@ export namespace entity {
),
};

addExcludeFromIndexes(entityObject.excludeFromIndexes, entityProto);

return entityProto;
}

export function addExcludeFromIndexes(
excludeFromIndexes: string[] | undefined,
entityProto: EntityProto
): EntityProto {
if (excludeFromIndexes && excludeFromIndexes.length > 0) {
excludeFromIndexes.forEach((excludePath: string) => {
excludePathFromEntity(entityProto, excludePath);
Expand Down Expand Up @@ -1463,7 +1471,7 @@ export interface ResponseResult {

export interface EntityObject {
data: {[k: string]: Entity};
excludeFromIndexes: string[];
excludeFromIndexes?: string[];
}

export interface Json {
Expand Down
45 changes: 33 additions & 12 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,10 @@ import {Transaction} from './transaction';
import {promisifyAll} from '@google-cloud/promisify';
import {google} from '../protos/protos';
import {AggregateQuery} from './aggregate';
import {SaveEntity} from './interfaces/save';

const {grpc} = new GrpcClient();
const addExcludeFromIndexes = entity.addExcludeFromIndexes;

export type PathType = string | number | entity.Int;
export interface BooleanObject {
Expand Down Expand Up @@ -1076,7 +1078,7 @@ class Datastore extends DatastoreRequest {
gaxOptionsOrCallback?: CallOptions | SaveCallback,
cb?: SaveCallback
): void | Promise<SaveResponse> {
entities = arrify(entities);
entities = arrify(entities) as SaveEntity[];
const gaxOptions =
typeof gaxOptionsOrCallback === 'object' ? gaxOptionsOrCallback : {};
const callback =
Expand Down Expand Up @@ -1109,22 +1111,30 @@ class Datastore extends DatastoreRequest {
}
}

if (entityObject.excludeLargeProperties) {
entityObject.excludeFromIndexes = entity.findLargeProperties_(
entityObject.data,
'',
entityObject.excludeFromIndexes
);
}

if (!entity.isKeyComplete(entityObject.key)) {
insertIndexes[index] = true;
}

// @TODO remove in @google-cloud/datastore@2.0.0
// This was replaced with a more efficient mechanism in the top-level
// `excludeFromIndexes` option.
if (Array.isArray(entityObject.data)) {
// This code populates the excludeFromIndexes list with the right values.
if (entityObject.excludeLargeProperties) {
entityObject.data.forEach(
(data: {
name: {
toString(): string;
};
value: Entity;
excludeFromIndexes?: boolean;
}) => {
entityObject.excludeFromIndexes = entity.findLargeProperties_(
data.value,
data.name.toString(),
entityObject.excludeFromIndexes
);
}
);
}
// This code builds the right entityProto from the entityObject
entityProto.properties = entityObject.data.reduce(
(
acc: EntityProtoReduceAccumulator,
Expand Down Expand Up @@ -1155,7 +1165,18 @@ class Datastore extends DatastoreRequest {
},
{}
);
// This code adds excludeFromIndexes in the right places
addExcludeFromIndexes(entityObject.excludeFromIndexes, entityProto);
} else {
// This code populates the excludeFromIndexes list with the right values.
if (entityObject.excludeLargeProperties) {
entityObject.excludeFromIndexes = entity.findLargeProperties_(
entityObject.data,
'',
entityObject.excludeFromIndexes
);
}
// This code builds the right entityProto from the entityObject
entityProto = entity.entityToEntityProto(entityObject);
}

Expand Down
74 changes: 74 additions & 0 deletions src/interfaces/save.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {Entity, entity} from '../entity';

/*
Entity data passed into save in non array form will be of type SaveNonArrayData
and does not require name and value properties.
*/
type SaveNonArrayData = {
[k: string]: Entity;
};

/*
Entity data passed into save in an array form will be of type SaveArrayData
so will have name and value defined because the source code of save requires a
name and a value to be defined or else an error will be thrown.
*/
interface SaveArrayData {
name: {
toString(): string;
};
value: Entity;
excludeFromIndexes?: boolean;
}

/*
When saving an entity, data in the data property of the entity is of type
SaveDataValue. The data can either be in array form in which case it will
match the SaveArrayData[] data type or it can be in non-array form where
it will match the SaveNonArrayData data type.
*/
export type SaveDataValue = SaveArrayData[] | SaveNonArrayData;

/*
An Entity passed into save will include a Key object contained either inside
a `key` property or inside a property indexed by the Key Symbol. If it is
the former then it will be of type SaveEntityWithoutKeySymbol.
*/
interface SaveEntityWithoutKeySymbol {
key: entity.Key;
data: SaveDataValue;
excludeFromIndexes?: string[];
}

/*
An Entity passed into save will include a Key object contained either inside
a `key` property or inside a property indexed by the Key Symbol. If it is
the latter then it will be of type SaveEntityWithKeySymbol.
*/
interface SaveEntityWithKeySymbol {
[entity.KEY_SYMBOL]: entity.Key;
data: SaveDataValue;
}

/*
Entities passed into the first argument of the save function are expected to be
of type SaveEntity[] after being turned into an array. We could change the
signature of save later to enforce this, but doing so would be a breaking change
so we just cast this value to SaveEntity[] for now to enable strong type
enforcement throughout this function.
*/
export type SaveEntity = SaveEntityWithoutKeySymbol | SaveEntityWithKeySymbol;
7 changes: 5 additions & 2 deletions src/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import * as protos from '../protos/protos';
import {serializer} from 'google-gax';
import * as gax from 'google-gax';
import {SaveDataValue} from './interfaces/save';
type JSONValue =
| string
| number
Expand Down Expand Up @@ -684,7 +685,7 @@
callback(null, isSingleLookup ? results[0] : results);
})
);
} catch (err: any) {

Check warning on line 688 in src/request.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
callback(err);
}
}
Expand Down Expand Up @@ -765,7 +766,7 @@
try {
sharedQueryOpts = this.getQueryOptions(query.query, options);
throwOnTransactionErrors(this, sharedQueryOpts);
} catch (error: any) {

Check warning on line 769 in src/request.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
callback(error);
return;
}
Expand Down Expand Up @@ -1412,8 +1413,10 @@
[key: string]: google.datastore.v1.Key | undefined;
}
export interface PrepareEntityObjectResponse {
key?: google.datastore.v1.Key;
data?: google.datastore.v1.Entity;
key?: entity.Key;
data?: SaveDataValue;
excludeFromIndexes?: string[];
excludeLargeProperties?: boolean;
method?: string;
}
export interface RequestCallback {
Expand Down
Loading
Loading