diff --git a/.github/workflows/java-server-sdk.yml b/.github/workflows/java-server-sdk.yml index 765d4281..f7131c4a 100644 --- a/.github/workflows/java-server-sdk.yml +++ b/.github/workflows/java-server-sdk.yml @@ -31,6 +31,13 @@ jobs: steps: - uses: actions/checkout@v3 + # Start the stores first. They are ready by the time the build finishes. + - uses: launchdarkly/gh-actions/actions/persistent-stores@persistent-stores-v0 + with: + redis: true + dynamodb: true + consul: true + - name: Shared CI Steps uses: ./.github/actions/ci with: @@ -43,26 +50,27 @@ jobs: - name: Start Contract Test Service shell: bash - run: make start-contract-test-service-bg -C lib/sdk/server + # The harness expects DynamoDB on port 8000, so the test service uses port 9000. + run: make start-contract-test-service-bg -C lib/sdk/server TEST_SERVICE_PORT=9000 - name: Contract Tests (v2) uses: launchdarkly/gh-actions/actions/contract-tests@contract-tests-v1 with: - test_service_port: '8000' + test_service_port: '9000' token: ${{ secrets.GITHUB_TOKEN }} debug_logging: 'true' stop_service: 'false' - enable_persistence_tests: 'false' + enable_persistence_tests: 'true' extra_params: '-skip-from=lib/sdk/server/contract-tests/test-suppressions.txt' - name: Contract Tests (v3) uses: launchdarkly/gh-actions/actions/contract-tests@contract-tests-v1 with: - test_service_port: '8000' + test_service_port: '9000' version: 'v3' token: ${{ secrets.GITHUB_TOKEN }} debug_logging: 'true' - enable_persistence_tests: 'false' + enable_persistence_tests: 'true' extra_params: '-skip-from=lib/sdk/server/contract-tests/test-suppressions-fdv2.txt' test-packaging: diff --git a/.github/workflows/nightly-contract-tests.yml b/.github/workflows/nightly-contract-tests.yml index 9700b780..d1940219 100644 --- a/.github/workflows/nightly-contract-tests.yml +++ b/.github/workflows/nightly-contract-tests.yml @@ -28,6 +28,13 @@ jobs: with: ref: ${{ inputs.branch || github.ref }} + # Start the stores first. They are ready by the time the build finishes. + - uses: launchdarkly/gh-actions/actions/persistent-stores@persistent-stores-v0 + with: + redis: true + dynamodb: true + consul: true + - name: Shared CI Steps uses: ./.github/actions/ci with: @@ -40,7 +47,8 @@ jobs: - name: Start Contract Test Service shell: bash - run: make start-contract-test-service-bg -C lib/sdk/server + # The harness expects DynamoDB on port 8000, so the test service uses port 9000. + run: make start-contract-test-service-bg -C lib/sdk/server TEST_SERVICE_PORT=9000 - name: Create JUnit output directory shell: bash @@ -49,21 +57,21 @@ jobs: - name: Contract Tests (v2) uses: launchdarkly/gh-actions/actions/contract-tests@contract-tests-v1 with: - test_service_port: '8000' + test_service_port: '9000' token: ${{ secrets.GITHUB_TOKEN }} debug_logging: 'true' stop_service: 'false' - enable_persistence_tests: 'false' + enable_persistence_tests: 'true' extra_params: '-skip-from=lib/sdk/server/contract-tests/test-suppressions.txt -enable-long-running-tests -junit junit/contract-tests-v2.xml' - name: Contract Tests v3 (with long-running tests) uses: launchdarkly/gh-actions/actions/contract-tests@contract-tests-v1 with: - test_service_port: '8000' + test_service_port: '9000' version: 'v3' token: ${{ secrets.GITHUB_TOKEN }} debug_logging: 'true' - enable_persistence_tests: 'false' + enable_persistence_tests: 'true' extra_params: '-skip-from=lib/sdk/server/contract-tests/test-suppressions-fdv2.txt -enable-long-running-tests -junit junit/contract-tests-v3.xml' - name: Upload contract test results diff --git a/lib/sdk/server/contract-tests/README.md b/lib/sdk/server/contract-tests/README.md index aa3942b8..7f2a5685 100644 --- a/lib/sdk/server/contract-tests/README.md +++ b/lib/sdk/server/contract-tests/README.md @@ -5,3 +5,5 @@ This directory contains an implementation of the cross-platform SDK testing prot To run these tests locally, run `make contract-tests` from the SDK project root directory. This downloads the correct version of the test harness tool automatically. Or, to test against an in-progress local version of the test harness, run `make start-contract-test-service` from the SDK project root directory; then, in the root directory of the `sdk-test-harness` project, build the test harness and run it from the command line. + +The persistent data store tests need Redis on port 6379, DynamoDB Local on port 8000, and Consul on port 8500. The harness runs them only with its `-enable-persistence-tests` option. DynamoDB uses the default port of the test service, so start the service on another port, for example `make start-contract-test-service TEST_SERVICE_PORT=9000`. diff --git a/lib/sdk/server/contract-tests/service/build.gradle b/lib/sdk/server/contract-tests/service/build.gradle index 80632939..97fd55d3 100644 --- a/lib/sdk/server/contract-tests/service/build.gradle +++ b/lib/sdk/server/contract-tests/service/build.gradle @@ -31,6 +31,9 @@ ext.versions = [ "logback": "1.1.3", "okhttp": "4.5.0", "testHelpers": "2.1.0", + "redisStore": "3.2.0", + "dynamodbStore": "5.0.1", + "consulStore": "5.0.0", "launchdarklyJavaSdkCommon": project(":sdk").versions["launchdarklyJavaSdkCommon"] ] @@ -45,4 +48,15 @@ dependencies { implementation "com.google.code.gson:gson:${versions.gson}" implementation "com.squareup.okhttp3:okhttp:${versions.okhttp}" implementation "com.launchdarkly:test-helpers:${versions.testHelpers}" + + // The store integrations depend on a published SDK. Exclude it, so that the tests use the SDK in this repository. + implementation("com.launchdarkly:launchdarkly-java-server-sdk-redis-store:${versions.redisStore}") { + exclude group: "com.launchdarkly", module: "launchdarkly-java-server-sdk" + } + implementation("com.launchdarkly:launchdarkly-java-server-sdk-dynamodb-store:${versions.dynamodbStore}") { + exclude group: "com.launchdarkly", module: "launchdarkly-java-server-sdk" + } + implementation("com.launchdarkly:launchdarkly-java-server-sdk-consul-store:${versions.consulStore}") { + exclude group: "com.launchdarkly", module: "launchdarkly-java-server-sdk" + } } diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java index 8d06694f..948f884a 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java @@ -35,6 +35,7 @@ public static class SdkConfigParams { SdkConfigProxyParams proxy; SdkConfigServiceEndpointParams serviceEndpoints; SdkConfigHookParams hooks; + SdkConfigPersistentDataStoreParams persistentDataStore; SdkConfigDataSystemParams dataSystem; } diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java index b6d32514..133f4936 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java @@ -30,6 +30,15 @@ import com.launchdarkly.sdk.server.integrations.StreamingDataSourceBuilder; import com.launchdarkly.sdk.server.integrations.PollingDataSourceBuilder; import com.launchdarkly.sdk.server.integrations.DataSystemBuilder; +import com.launchdarkly.sdk.server.integrations.Consul; +import com.launchdarkly.sdk.server.integrations.ConsulDataStoreBuilder; +import com.launchdarkly.sdk.server.integrations.DynamoDb; +import com.launchdarkly.sdk.server.integrations.DynamoDbStoreBuilder; +import com.launchdarkly.sdk.server.integrations.PersistentDataStoreBuilder; +import com.launchdarkly.sdk.server.integrations.Redis; +import com.launchdarkly.sdk.server.integrations.RedisStoreBuilder; +import com.launchdarkly.sdk.server.subsystems.DataSystemConfiguration; +import com.launchdarkly.sdk.server.subsystems.PersistentDataStore; import com.launchdarkly.sdk.server.DataSystemComponents; import com.launchdarkly.sdk.server.integrations.FDv2PollingInitializerBuilder; import com.launchdarkly.sdk.server.integrations.FDv2PollingSynchronizerBuilder; @@ -45,6 +54,11 @@ import org.slf4j.LoggerFactory; import java.net.URI; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +import java.net.MalformedURLException; import java.net.URL; import java.time.Duration; import java.util.ArrayList; @@ -77,7 +91,12 @@ import sdktest.Representations.UnregisterListenerParams; import sdktest.Representations.HookConfig; import sdktest.Representations.SdkConfigHookParams; +import sdktest.Representations.PersistentCacheMode; +import sdktest.Representations.PersistentStoreType; import sdktest.Representations.SdkConfigParams; +import sdktest.Representations.SdkConfigPersistentDataStoreParams; +import sdktest.Representations.SdkConfigPersistentStoreParams; +import sdktest.Representations.StoreMode; import sdktest.Representations.SdkConfigDataSystemParams; import sdktest.Representations.SdkConfigDataInitializerParams; import sdktest.Representations.SdkConfigSynchronizerParams; @@ -573,41 +592,25 @@ private LDConfig buildSdkConfig(SdkConfigParams params, String tag) { builder.hooks(Components.hooks().setHooks(hookList)); } + if (params.persistentDataStore != null) { + builder.dataStore(buildPersistentStore(params.persistentDataStore)); + // With a store and no data source, the SDK reads only from the store. + if (params.streaming == null && params.polling == null && params.serviceEndpoints == null + && params.dataSystem == null) { + builder.dataSource(Components.externalUpdatesOnly()); + } + } + if (params.dataSystem != null) { DataSystemBuilder dataSystemBuilder = Components.dataSystem().custom(); - // TODO: enable this code in the future and determine which dependencies on persistent stores need to be added to contract test build process - // Configure persistent store if provided - // if (params.dataSystem.store != null && params.dataSystem.store.persistentDataStore != null) { - // var storeConfig = params.dataSystem.store.persistentDataStore; - // var storeType = storeConfig.store.type.toLowerCase(); - // ComponentConfigurer persistentStore = null; - // - // switch (storeType) { - // case "redis": - // // Redis store configuration - // break; - // case "dynamodb": - // // DynamoDB store configuration - // break; - // case "consul": - // // Consul store configuration - // break; - // } - // - // if (persistentStore != null) { - // // Configure cache - // var cacheMode = storeConfig.cache != null ? storeConfig.cache.mode.toLowerCase() : null; - // // ... cache configuration ... - // - // // Determine store mode - // var storeMode = params.dataSystem.storeMode == 0 - // ? DataSystemConfiguration.DataStoreMode.READ_ONLY - // : DataSystemConfiguration.DataStoreMode.READ_WRITE; - // - // dataSystemBuilder.persistentStore(persistentStore, storeMode); - // } - // } + if (params.dataSystem.store != null && params.dataSystem.store.persistentDataStore != null) { + DataSystemConfiguration.DataStoreMode storeMode = + params.dataSystem.storeMode != null && params.dataSystem.storeMode == StoreMode.READ_WRITE + ? DataSystemConfiguration.DataStoreMode.READ_WRITE + : DataSystemConfiguration.DataStoreMode.READ_ONLY; + dataSystemBuilder.persistentStore(buildPersistentStore(params.dataSystem.store.persistentDataStore), storeMode); + } // Configure initializers if (params.dataSystem.initializers != null && params.dataSystem.initializers.length > 0) { @@ -670,6 +673,66 @@ private LDConfig buildSdkConfig(SdkConfigParams params, String tag) { return builder.build(); } + private static PersistentDataStoreBuilder buildPersistentStore(SdkConfigPersistentDataStoreParams params) { + SdkConfigPersistentStoreParams store = params.store; + PersistentDataStoreBuilder builder; + switch (store.type) { + case PersistentStoreType.REDIS: { + RedisStoreBuilder redis = Redis.dataStore().uri(URI.create(store.dsn)); + if (store.prefix != null) { + redis.prefix(store.prefix); + } + builder = Components.persistentDataStore(redis); + break; + } + case PersistentStoreType.DYNAMODB: { + // The harness creates this table in its local DynamoDB. + DynamoDbStoreBuilder dynamo = DynamoDb.dataStore("sdk-contract-tests") + .endpoint(URI.create(store.dsn)) + .region(Region.US_EAST_1) + .credentials(StaticCredentialsProvider.create(AwsBasicCredentials.create("dummy", "dummy"))); + if (store.prefix != null) { + dynamo.prefix(store.prefix); + } + builder = Components.persistentDataStore(dynamo); + break; + } + case PersistentStoreType.CONSUL: { + // The harness sends the Consul agent address as host:port, without a scheme. + ConsulDataStoreBuilder consul; + try { + consul = Consul.dataStore().url(new URL("http://" + store.dsn)); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("invalid Consul address: " + store.dsn, e); + } + if (store.prefix != null) { + consul.prefix(store.prefix); + } + builder = Components.persistentDataStore(consul); + break; + } + default: + throw new IllegalArgumentException("unsupported persistent store type: " + store.type); + } + + if (params.cache != null && params.cache.mode != null) { + switch (params.cache.mode) { + case PersistentCacheMode.OFF: + builder.noCaching(); + break; + case PersistentCacheMode.TTL: + builder.cacheSeconds(params.cache.ttl); + break; + case PersistentCacheMode.INFINITE: + builder.cacheForever(); + break; + default: + throw new IllegalArgumentException("unsupported persistent cache mode: " + params.cache.mode); + } + } + return builder; + } + private DataSourceBuilder createSynchronizer( SdkConfigSynchronizerParams synchronizer, String payloadFilter) { diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java index cbc0fab9..325f9002 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java @@ -21,7 +21,8 @@ import sdktest.Representations.Status; public class TestService { - private static final int PORT = 8000; + // The PORT environment variable sets the port. The Makefile sets it from TEST_SERVICE_PORT. + private static final int PORT = System.getenv("PORT") != null ? Integer.parseInt(System.getenv("PORT")) : 8000; private static final String[] CAPABILITIES = new String[]{ "all-flags-client-side-only", "all-flags-details-only-for-tracked-flags", @@ -41,6 +42,9 @@ public class TestService { "http-proxy", "inline-context-all", "migrations", + "persistent-data-store-consul", + "persistent-data-store-dynamodb", + "persistent-data-store-redis", "optional-event-gzip", "secure-mode-hash", "server-side", diff --git a/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt b/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt index 9765a098..ddf295e7 100644 --- a/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt +++ b/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt @@ -1,3 +1,22 @@ streaming/fdv2/recoverable fallback to secondary synchronizer streaming/fdv2/recoverable fallback with recovery streaming/fdv2/permanent fallback with recovery +persistent data store/consul/collection cardinality +persistent data store/consul/store mode read - no data source +persistent data store/consul/store mode read-write - no data source +persistent data store/consul/tombstones +persistent data store/consul/uses custom prefix +persistent data store/consul/uses default prefix +persistent data store/dynamodb/collection cardinality +persistent data store/dynamodb/read-write/data source deletions respect versioning +persistent data store/dynamodb/store mode read - no data source +persistent data store/dynamodb/store mode read-write - no data source +persistent data store/dynamodb/tombstones +persistent data store/dynamodb/uses custom prefix +persistent data store/dynamodb/uses default prefix +persistent data store/redis/collection cardinality +persistent data store/redis/store mode read - no data source +persistent data store/redis/store mode read-write - no data source +persistent data store/redis/tombstones +persistent data store/redis/uses custom prefix +persistent data store/redis/uses default prefix diff --git a/lib/sdk/server/contract-tests/test-suppressions.txt b/lib/sdk/server/contract-tests/test-suppressions.txt index 8b137891..790c805d 100644 --- a/lib/sdk/server/contract-tests/test-suppressions.txt +++ b/lib/sdk/server/contract-tests/test-suppressions.txt @@ -1 +1,11 @@ +persistent data store/consul/daemon mode +persistent data store/consul/uses custom prefix +persistent data store/consul/uses default prefix +persistent data store/dynamodb/daemon mode +persistent data store/dynamodb/read-write/data source deletions respect versioning +persistent data store/dynamodb/uses custom prefix +persistent data store/dynamodb/uses default prefix +persistent data store/redis/daemon mode +persistent data store/redis/uses custom prefix +persistent data store/redis/uses default prefix