diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 665f8283e..d28da4433 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -106,6 +106,12 @@ class BuildConfig { @Value('${wave.build.retry-attempts:0}') int retryAttempts + @Value('${wave.build.max-conda-file-size:50000}') + int maxCondaFileSize + + @Value('${wave.build.max-container-file-size:10000}') + int maxContainerFileSize + @PostConstruct private void init() { log.info("Builder config: " + diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 355a8d299..f42ffc08f 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -278,8 +278,8 @@ class ContainerController { protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequestData data, TokenData token, String target, String ip) { try { - final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration) - persistenceService.saveContainerRequest(token.value, recrd) + final recrd = new WaveContainerRecord(req, data, token.value, target, ip, token.expiration) + persistenceService.saveContainerRequest(recrd) } catch (Throwable e) { log.error("Unable to store container request with token: ${token}", e) @@ -522,6 +522,12 @@ class ContainerController { void validateContainerRequest(SubmitContainerTokenRequest req) throws BadRequestException { String msg + //check conda file size + if( req.condaFile && req.condaFile.length() > buildConfig.maxCondaFileSize ) + throw new BadRequestException("Conda file size exceeds the maximum allowed size of ${buildConfig.maxCondaFileSize} bytes") + // check container file size + if( req.containerFile && req.containerFile.length() > buildConfig.maxContainerFileSize ) + throw new BadRequestException("Container file size exceeds the maximum allowed size of ${buildConfig.maxContainerFileSize} bytes") // check valid image name msg = validationService.checkContainerName(req.containerImage) if( msg ) throw new BadRequestException(msg) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 5cd103f93..23dd2fcf9 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -68,10 +68,9 @@ interface PersistenceService { /** * Store a {@link WaveContainerRecord} object in the Surreal wave_request table. * - * @param token The request token associated with this request * @param data A {@link WaveContainerRecord} object representing a Wave request record */ - void saveContainerRequest(String token, WaveContainerRecord data) + void saveContainerRequest(WaveContainerRecord data) /** * Update a container request with the digest of the resolved request diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index c6070f182..67a402ac7 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.api.FusionVersion import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.tower.User @@ -42,6 +41,12 @@ import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime @CompileStatic class WaveContainerRecord { + /** + * wave request id, this will be the token + * This is container token and it is named as id for surrealdb requirement + */ + final String id + /** * The Tower user associated with the request */ @@ -158,7 +163,8 @@ class WaveContainerRecord { */ final String fusionVersion - WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequestData data, String waveImage, String addr, Instant expiration) { + WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequestData data, String token, String waveImage, String addr, Instant expiration) { + this.id = token this.user = data.identity.user this.workspaceId = request.towerWorkspaceId this.containerImage = request.containerImage @@ -184,6 +190,7 @@ class WaveContainerRecord { } WaveContainerRecord(WaveContainerRecord that, String sourceDigest, String waveDigest) { + this.id = that.id this.user = that.user this.workspaceId = that.workspaceId this.containerImage = that.containerImage diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy index 0c24dbaf6..109018f6c 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy @@ -66,8 +66,8 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveContainerRequest(String token, WaveContainerRecord data) { - requestStore.put(token, data) + void saveContainerRequest(WaveContainerRecord data) { + requestStore.put(data.id, data) } @Override diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy index def12de82..5e6ea24c4 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy @@ -24,14 +24,12 @@ import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Get import io.micronaut.http.annotation.Header import io.micronaut.http.annotation.Post -import io.micronaut.http.annotation.Put import io.micronaut.http.client.annotation.Client import io.micronaut.retry.annotation.Retryable import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord import reactor.core.publisher.Flux /** * Declarative http client for SurrealDB @@ -74,12 +72,6 @@ interface SurrealClient { @Get('/key/wave_request/{token}') String getContainerRequest(@Header String authorization, String token) - @Post('/key/wave_request/{token}') - Flux> insertContainerRequestAsync(@Header String authorization, String token, @Body WaveContainerRecord body) - - @Put('/key/wave_request/{token}') - Flux> updateContainerRequestAsync(@Header String authorization, String token, @Body WaveContainerRecord body) - @Post('/key/wave_scan') Map insertScanRecord(@Header String authorization, @Body WaveScanRecord body) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index d475b8646..7c7167387 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -100,15 +100,21 @@ class SurrealPersistenceService implements PersistenceService { @Override void saveBuild(WaveBuildRecord build) { - surrealDb.insertBuildAsync(getAuthorization(), build).subscribe({ result-> - log.trace "Build request with id '$build.buildId' saved record: ${result}" - }, {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving Build request record ${msg}\n${build}", error) - }) + // note: use surreal sql in order to by-pass issue with large payload + // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 + final query = "INSERT INTO wave_build ${JacksonHelper.toJson(build)}" + surrealDb + .sqlAsync(getAuthorization(), query) + .subscribe({result -> + log.trace "Build request with id '$build.buildId' saved record: ${result}" + }, + {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error saving Build request record ${msg}\n${build}", error) + }) } @Override @@ -167,16 +173,22 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveContainerRequest(String token, WaveContainerRecord data) { - surrealDb.insertContainerRequestAsync(authorization, token, data).subscribe({ result-> - log.trace "Container request with token '$token' saved record: ${result}" - }, {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving container request record ${msg}\n${data}", error) - }) + void saveContainerRequest(WaveContainerRecord data) { + // note: use surreal sql in order to by-pass issue with large payload + // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 + final query = "INSERT INTO wave_request ${JacksonHelper.toJson(data)}" + surrealDb + .sqlAsync(getAuthorization(), query) + .subscribe({result -> + log.trace "Container request with token '$data.id' saved record: ${result}" + }, + {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error saving container request record ${msg}\n${data}", error) + }) } void updateContainerRequest(String token, ContainerDigestPair digest) { @@ -206,11 +218,15 @@ class SurrealPersistenceService implements PersistenceService { final json = surrealDb.getContainerRequest(getAuthorization(), token) log.trace "Container request with token '$token' loaded: ${json}" final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null + final data= json ? JacksonHelper.fromJson(patchSurrealId(json,"wave_request"), type) : null final result = data && data[0].result ? data[0].result[0] : null return result } + static protected String patchSurrealId(String json, String table) { + json.replaceFirst(/"id":\s*"${table}:(\w*)"/) { List it-> /"id":"${it[1]}"/ } + } + void createScanRecord(WaveScanRecord scanRecord) { final result = surrealDb.insertScanRecord(authorization, scanRecord) log.trace "Scan create result=$result" diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index 499733dac..f48b1505f 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -194,11 +194,11 @@ class ViewControllerTest extends Specification { and: def exp = Instant.now().plusSeconds(3600) - def container = new WaveContainerRecord(req, data, wave, addr, exp) def token = '12345' + def container = new WaveContainerRecord(req, data, token, wave, addr, exp) when: - persistenceService.saveContainerRequest(token, container) + persistenceService.saveContainerRequest(container) and: def request = HttpRequest.GET("/view/containers/${token}") def response = client.toBlocking().exchange(request, String) diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy index 4aa2c3fa3..035586280 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy @@ -59,8 +59,10 @@ class WaveContainerRecordTest extends Specification { when: def exp = Instant.now().plusSeconds(3600) - def container = new WaveContainerRecord(req, data, wave, addr, exp) + def token = '1234' + def container = new WaveContainerRecord(req, data, token, wave, addr, exp) then: + container.id == token container.user == user container.workspaceId == req.towerWorkspaceId container.containerImage == req.containerImage @@ -101,8 +103,10 @@ class WaveContainerRecordTest extends Specification { when: def exp = Instant.now().plusSeconds(3600) - def container = new WaveContainerRecord(req, data, wave, addr, exp) + def token = '1234' + def container = new WaveContainerRecord(req, data, token, wave, addr, exp) then: + container.id == token container.user == user container.workspaceId == req.towerWorkspaceId container.containerImage == req.containerImage diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index af1f812b8..cdbd876aa 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -46,6 +46,8 @@ import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.test.SurrealDBTestContainer import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User +import org.apache.commons.lang3.RandomStringUtils + /** * @author : jorge * @@ -233,6 +235,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def 'should load a request record' () { given: + def largeContainerFile = RandomStringUtils.random(25600, true, true) def persistence = applicationContext.getBean(SurrealPersistenceService) and: def TOKEN = '123abc' @@ -249,15 +252,15 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe timestamp: Instant.now().toString() ) def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world' ) + def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world', largeContainerFile ) def wave = "wave.io/wt/$TOKEN/hello-world" def addr = "100.200.300.400" def exp = Instant.now().plusSeconds(3600) and: - def request = new WaveContainerRecord(req, data, wave, addr, exp) + def request = new WaveContainerRecord(req, data, TOKEN, wave, addr, exp) and: - persistence.saveContainerRequest(TOKEN, request) + persistence.saveContainerRequest(request) and: sleep 200 // <-- the above request is async, give time to save it @@ -265,7 +268,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def loaded = persistence.loadContainerRequest(TOKEN) then: loaded == request - + loaded.containerFile == largeContainerFile // should update the record when: @@ -366,5 +369,47 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe stored == result } + def 'should remove surreal table from json' () { + given: + def json = /{"id":"wave_request:1234abc", "this":"one", "that":123 }/ + expect: + SurrealPersistenceService.patchSurrealId(json, "wave_request") + == /{"id":"1234abc", "this":"one", "that":123 }/ + } + + def 'should save 50KB container and conda file' (){ + given: + def data = RandomStringUtils.random(25600, true, true) + def persistence = applicationContext.getBean(SurrealPersistenceService) + final request = new BuildRequest( + 'container1234', + data, + data, + Path.of("/some/path"), + 'buildrepo:recipe-container1234', + PlatformId.NULL, + ContainerPlatform.of('amd64'), + 'docker.io/my/cache', + '127.0.0.1', + '{"config":"json"}', + null, + null, + 'scan12345', + null, + BuildFormat.DOCKER, + Duration.ofMinutes(1) + ).withBuildId('123') + and: + def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') + + and: + def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) + + when: + persistence.saveBuild(build1) + sleep 100 + then: + persistence.loadBuild(request.buildId) == build1 + } }