Skip to content

Commit 627615f

Browse files
algattikalgattik
authored
System tests: extend Gatling scope (#1389)
* Feature/1296/191 extend gatling scope (#263) * Update BlobTransferUtils.java * PR comments * Update AbstractAzureBlobTest.java * Created TestFunctions static class for getBlobServiceClient and associated methods
1 parent 5be3459 commit 627615f

16 files changed

Lines changed: 186 additions & 106 deletions

File tree

‎.github/workflows/performancetests.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
env:
1616
REPEAT: 5
1717
AT_ONCE_USERS: 10
18-
MAX_RESPONSE_TIME: 5000
18+
MAX_RESPONSE_TIME: 10000
1919
SUCCESS_PERCENTAGE: 100.0
2020

2121
steps:

‎extensions/azure/azure-test/src/testFixtures/java/org/eclipse/dataspaceconnector/azure/testfixtures/AbstractAzureBlobTest.java‎

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@
1616

1717
import com.azure.storage.blob.BlobContainerClient;
1818
import com.azure.storage.blob.BlobServiceClient;
19-
import com.azure.storage.blob.BlobServiceClientBuilder;
20-
import com.azure.storage.common.StorageSharedKeyCredential;
21-
import org.jetbrains.annotations.NotNull;
2219
import org.junit.jupiter.api.AfterEach;
2320
import org.junit.jupiter.api.BeforeEach;
2421

@@ -47,28 +44,12 @@ public abstract class AbstractAzureBlobTest {
4744
public void setupClient() {
4845
account1ContainerName = "storage-container-" + testRunId;
4946

50-
blobServiceClient1 = getBlobServiceClient(account1Name, account1Key);
51-
blobServiceClient2 = getBlobServiceClient(account2Name, account2Key);
47+
blobServiceClient1 = TestFunctions.getBlobServiceClient(account1Name, account1Key);
48+
blobServiceClient2 = TestFunctions.getBlobServiceClient(account2Name, account2Key);
5249

5350
createContainer(blobServiceClient1, account1ContainerName);
5451
}
5552

56-
@NotNull
57-
private BlobServiceClient getBlobServiceClient(String accountName, String key) {
58-
var client = new BlobServiceClientBuilder()
59-
.credential(new StorageSharedKeyCredential(accountName, key))
60-
.endpoint(getEndpoint(accountName))
61-
.buildClient();
62-
63-
client.getAccountInfo();
64-
return client;
65-
}
66-
67-
@NotNull
68-
protected String getEndpoint(String accountName) {
69-
return "http://127.0.0.1:10000/" + accountName;
70-
}
71-
7253
protected void createContainer(BlobServiceClient client, String containerName) {
7354
assertFalse(client.getBlobContainerClient(containerName).exists());
7455

‎extensions/azure/azure-test/src/testFixtures/java/org/eclipse/dataspaceconnector/azure/testfixtures/TestFunctions.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (c) 2022 Microsoft Corporation
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Apache License, Version 2.0 which is available at
6+
* https://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* SPDX-License-Identifier: Apache-2.0
9+
*
10+
* Contributors:
11+
* Microsoft Corporation - initial API and implementation
12+
*
13+
*/
14+
15+
package org.eclipse.dataspaceconnector.azure.testfixtures;
16+
17+
import com.azure.storage.blob.BlobServiceClient;
18+
import com.azure.storage.blob.BlobServiceClientBuilder;
19+
import com.azure.storage.common.StorageSharedKeyCredential;
20+
import org.jetbrains.annotations.NotNull;
21+
22+
public final class TestFunctions {
23+
private TestFunctions() {
24+
}
25+
26+
@NotNull
27+
public static BlobServiceClient getBlobServiceClient(String accountName, String key) {
28+
return getBlobServiceClient(accountName, key, getBlobServiceTestEndpoint(accountName));
29+
}
30+
31+
@NotNull
32+
public static BlobServiceClient getBlobServiceClient(String accountName, String key, String endpoint) {
33+
var client = new BlobServiceClientBuilder()
34+
.credential(new StorageSharedKeyCredential(accountName, key))
35+
.endpoint(endpoint)
36+
.buildClient();
37+
38+
client.getAccountInfo();
39+
return client;
40+
}
41+
42+
@NotNull
43+
public static String getBlobServiceTestEndpoint(String accountName) {
44+
return "http://127.0.0.1:10000/" + accountName;
45+
}
46+
}

‎extensions/azure/blobstorage/blob-provision/src/test/java/org/eclipse/dataspaceconnector/provision/azure/blob/ObjectContainerStatusCheckerIntegrationTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.eclipse.dataspaceconnector.azure.blob.core.AzureBlobStoreSchema;
1919
import org.eclipse.dataspaceconnector.azure.blob.core.api.BlobStoreApiImpl;
2020
import org.eclipse.dataspaceconnector.azure.testfixtures.AbstractAzureBlobTest;
21+
import org.eclipse.dataspaceconnector.azure.testfixtures.TestFunctions;
2122
import org.eclipse.dataspaceconnector.azure.testfixtures.annotations.AzureStorageIntegrationTest;
2223
import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils;
2324
import org.eclipse.dataspaceconnector.spi.security.Vault;
@@ -49,7 +50,7 @@ void setUp() {
4950
Vault vault = mock(Vault.class);
5051

5152
when(vault.resolveSecret(account1Name + "-key1")).thenReturn(account1Key);
52-
var blobStoreApi = new BlobStoreApiImpl(vault, getEndpoint(account1Name));
53+
var blobStoreApi = new BlobStoreApiImpl(vault, TestFunctions.getBlobServiceTestEndpoint(account1Name));
5354
checker = new ObjectContainerStatusChecker(blobStoreApi, policy);
5455
}
5556

‎extensions/azure/data-plane/storage/src/test/java/org/eclipse/dataspaceconnector/azure/dataplane/azurestorage/AzureDataPlaneCopyIntegrationTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.eclipse.dataspaceconnector.azure.dataplane.azurestorage.pipeline.AzureStorageDataSinkFactory;
2424
import org.eclipse.dataspaceconnector.azure.dataplane.azurestorage.pipeline.AzureStorageDataSourceFactory;
2525
import org.eclipse.dataspaceconnector.azure.testfixtures.AbstractAzureBlobTest;
26+
import org.eclipse.dataspaceconnector.azure.testfixtures.TestFunctions;
2627
import org.eclipse.dataspaceconnector.azure.testfixtures.annotations.AzureStorageIntegrationTest;
2728
import org.eclipse.dataspaceconnector.spi.monitor.Monitor;
2829
import org.eclipse.dataspaceconnector.spi.security.Vault;
@@ -64,8 +65,8 @@ class AzureDataPlaneCopyIntegrationTest extends AbstractAzureBlobTest {
6465
Monitor monitor = mock(Monitor.class);
6566
Vault vault = mock(Vault.class);
6667

67-
BlobStoreApi account1Api = new BlobStoreApiImpl(vault, getEndpoint(account1Name));
68-
BlobStoreApi account2Api = new BlobStoreApiImpl(vault, getEndpoint(account2Name));
68+
BlobStoreApi account1Api = new BlobStoreApiImpl(vault, TestFunctions.getBlobServiceTestEndpoint(account1Name));
69+
BlobStoreApi account2Api = new BlobStoreApiImpl(vault, TestFunctions.getBlobServiceTestEndpoint(account2Name));
6970

7071
@BeforeEach
7172
void setUp() {

‎system-tests/azure-data-factory-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/AzureDataFactoryTransferIntegrationTest.java‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,15 @@
3535
import java.util.ArrayList;
3636
import java.util.List;
3737
import java.util.Map;
38+
import java.util.Objects;
3839
import java.util.Properties;
3940
import java.util.UUID;
4041

42+
import static java.lang.String.format;
4143
import static java.lang.String.valueOf;
4244
import static java.lang.System.getenv;
4345
import static java.nio.charset.StandardCharsets.UTF_8;
44-
import static org.assertj.core.api.Assertions.assertThat;
46+
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferSimulationConfiguration.BLOB_CONTENT;
4547
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.createAsset;
4648
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.createContractDefinition;
4749
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.createPolicy;
@@ -135,13 +137,12 @@ static void cleanUp() {
135137
public void transferBlob_success() {
136138
// Arrange
137139
var vault = AzureVault.authenticateWithSecret(new ConsoleMonitor(), AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET, KEY_VAULT_NAME);
140+
var account2Key = Objects.requireNonNull(vault.resolveSecret(format("%s-key1", CONSUMER_STORAGE_ACCOUNT_NAME)));
138141
var blobStoreApi = new BlobStoreApiImpl(vault, BLOB_STORE_ENDPOINT_TEMPLATE);
139142

140143
// Upload a blob with test data on provider blob container
141-
var blobContent = "AzureDataFactoryTransferIntegrationTest-" + UUID.randomUUID();
142-
143144
blobStoreApi.createContainer(PROVIDER_STORAGE_ACCOUNT_NAME, PROVIDER_CONTAINER_NAME);
144-
blobStoreApi.putBlob(PROVIDER_STORAGE_ACCOUNT_NAME, PROVIDER_CONTAINER_NAME, PROVIDER_ASSET_FILE, blobContent.getBytes(UTF_8));
145+
blobStoreApi.putBlob(PROVIDER_STORAGE_ACCOUNT_NAME, PROVIDER_CONTAINER_NAME, PROVIDER_ASSET_FILE, BLOB_CONTENT.getBytes(UTF_8));
145146
// Add for cleanup
146147
CONTAINER_CLEANUP.add(() -> blobStoreApi.deleteContainer(PROVIDER_STORAGE_ACCOUNT_NAME, PROVIDER_CONTAINER_NAME));
147148

@@ -152,22 +153,10 @@ public void transferBlob_success() {
152153

153154
// Act
154155
System.setProperty(BlobTransferLocalSimulation.ACCOUNT_NAME_PROPERTY, CONSUMER_STORAGE_ACCOUNT_NAME);
156+
System.setProperty(BlobTransferLocalSimulation.ACCOUNT_KEY_PROPERTY, account2Key);
157+
System.setProperty(BlobTransferLocalSimulation.ACCOUNT_ENDPOINT_PROPERTY, format("https://%s.blob.core.windows.net", CONSUMER_STORAGE_ACCOUNT_NAME));
155158
System.setProperty(BlobTransferLocalSimulation.MAX_DURATION_SECONDS_PROPERTY, "360"); // ADF SLA is to initiate copy within 4 minutes
156159
runGatling(BlobTransferLocalSimulation.class, TransferSimulationUtils.DESCRIPTION);
157-
158-
// Assert
159-
var provisionedContainerName = BlobTransferUtils.getProvisionedContainerName();
160-
// Add for cleanup
161-
CONTAINER_CLEANUP.add(() -> blobStoreApi.deleteContainer(CONSUMER_STORAGE_ACCOUNT_NAME, provisionedContainerName));
162-
163-
var actualBlobContent = blobStoreApi.getBlob(CONSUMER_STORAGE_ACCOUNT_NAME, provisionedContainerName, PROVIDER_ASSET_FILE);
164-
assertThat(actualBlobContent.length)
165-
.withFailMessage("Destination blob %s not created", PROVIDER_ASSET_FILE)
166-
.isGreaterThan(0);
167-
assertThat(new String(actualBlobContent))
168-
.withFailMessage("Transferred file contents are not same as the source file")
169-
.isEqualTo(blobContent);
170-
171160
}
172161

173162
@NotNull

‎system-tests/azure-tests/build.gradle.kts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ plugins {
2222
val gatlingVersion: String by project
2323
val storageBlobVersion: String by project
2424
val restAssured: String by project
25+
val assertj: String by project
26+
val faker: String by project
2527

2628
dependencies {
2729
testImplementation("io.gatling.highcharts:gatling-charts-highcharts:${gatlingVersion}") {
@@ -41,8 +43,11 @@ dependencies {
4143
testImplementation(testFixtures(project(":common:util")))
4244
testImplementation(testFixtures(project(":launchers:junit")))
4345
testImplementation(testFixtures(project(":system-tests:tests")))
44-
testFixturesImplementation(testFixtures(project(":system-tests:tests")))
4546
testImplementation(testFixtures(project(":extensions:azure:azure-test")))
47+
testFixturesImplementation(testFixtures(project(":system-tests:tests")))
48+
testFixturesImplementation(testFixtures(project(":extensions:azure:azure-test")))
49+
testFixturesImplementation("org.assertj:assertj-core:${assertj}")
50+
testFixturesImplementation("com.github.javafaker:javafaker:${faker}")
4651
testImplementation("com.azure:azure-storage-blob:${storageBlobVersion}")
4752
testFixturesImplementation("io.rest-assured:rest-assured:${restAssured}")
4853

‎system-tests/azure-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferIntegrationTest.java‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import com.azure.core.util.BinaryData;
2020
import org.eclipse.dataspaceconnector.azure.testfixtures.AbstractAzureBlobTest;
21+
import org.eclipse.dataspaceconnector.azure.testfixtures.TestFunctions;
2122
import org.eclipse.dataspaceconnector.azure.testfixtures.annotations.AzureStorageIntegrationTest;
2223
import org.eclipse.dataspaceconnector.junit.launcher.EdcRuntimeExtension;
2324
import org.eclipse.dataspaceconnector.junit.launcher.MockVault;
@@ -35,12 +36,12 @@
3536
import java.util.UUID;
3637

3738
import static java.lang.String.format;
38-
import static org.assertj.core.api.Assertions.assertThat;
39+
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferLocalSimulation.ACCOUNT_ENDPOINT_PROPERTY;
40+
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferLocalSimulation.ACCOUNT_KEY_PROPERTY;
3941
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferLocalSimulation.ACCOUNT_NAME_PROPERTY;
4042
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.createAsset;
4143
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.createContractDefinition;
4244
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.createPolicy;
43-
import static org.eclipse.dataspaceconnector.system.tests.local.BlobTransferUtils.getProvisionedContainerName;
4445
import static org.eclipse.dataspaceconnector.system.tests.local.TransferLocalSimulation.CONSUMER_CONNECTOR_PATH;
4546
import static org.eclipse.dataspaceconnector.system.tests.local.TransferLocalSimulation.CONSUMER_CONNECTOR_PORT;
4647
import static org.eclipse.dataspaceconnector.system.tests.local.TransferLocalSimulation.CONSUMER_IDS_API;
@@ -109,7 +110,7 @@ private static void setUpMockVault(EdcRuntimeExtension consumer, Vault vault) {
109110
public void transferBlob_success() {
110111
// Arrange
111112
// Upload a blob with test data on provider blob container (in account1).
112-
var blobContent = "BlobTransferIntegrationTest-" + UUID.randomUUID();
113+
var blobContent = BlobTransferSimulationConfiguration.BLOB_CONTENT;
113114
createContainer(blobServiceClient1, PROVIDER_CONTAINER_NAME);
114115
blobServiceClient1.getBlobContainerClient(PROVIDER_CONTAINER_NAME)
115116
.getBlobClient(PROVIDER_ASSET_FILE)
@@ -126,19 +127,9 @@ public void transferBlob_success() {
126127

127128
// Act
128129
System.setProperty(ACCOUNT_NAME_PROPERTY, account2Name);
130+
System.setProperty(ACCOUNT_KEY_PROPERTY, account2Key);
131+
System.setProperty(ACCOUNT_ENDPOINT_PROPERTY, TestFunctions.getBlobServiceTestEndpoint(account2Name));
129132
runGatling(BlobTransferLocalSimulation.class, TransferSimulationUtils.DESCRIPTION);
130-
131-
// Assert
132-
var container = getProvisionedContainerName();
133-
var destinationBlob = blobServiceClient2.getBlobContainerClient(container)
134-
.getBlobClient(PROVIDER_ASSET_FILE);
135-
assertThat(destinationBlob.exists())
136-
.withFailMessage("Destination blob %s not created", destinationBlob.getBlobUrl())
137-
.isTrue();
138-
var actualBlobContent = destinationBlob.downloadContent().toString();
139-
assertThat(actualBlobContent)
140-
.withFailMessage("Transferred file contents are not same as the source file")
141-
.isEqualTo(blobContent);
142133
}
143134

144135
}

‎system-tests/azure-tests/src/testFixtures/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferLocalSimulation.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,15 @@
1616

1717
public class BlobTransferLocalSimulation extends TransferLocalSimulation {
1818
static final String ACCOUNT_NAME_PROPERTY = "BlobTransferLocalSimulation-account-name";
19+
static final String ACCOUNT_KEY_PROPERTY = "BlobTransferLocalSimulation-account-key";
20+
static final String ACCOUNT_ENDPOINT_PROPERTY = "BlobTransferLocalSimulation-account-endpoint";
1921
static final String MAX_DURATION_SECONDS_PROPERTY = "BlobTransferLocalSimulation-copy-max-duration-seconds";
2022

2123
public BlobTransferLocalSimulation() {
22-
super(new BlobTransferSimulationConfiguration(System.getProperty(ACCOUNT_NAME_PROPERTY),
24+
super(new BlobTransferSimulationConfiguration(
25+
System.getProperty(ACCOUNT_NAME_PROPERTY),
26+
System.getProperty(ACCOUNT_KEY_PROPERTY),
27+
System.getProperty(ACCOUNT_ENDPOINT_PROPERTY),
2328
Integer.parseInt(System.getProperty(MAX_DURATION_SECONDS_PROPERTY, "30"))));
2429
}
2530
}

‎system-tests/azure-tests/src/testFixtures/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferSimulationConfiguration.java‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414

1515
package org.eclipse.dataspaceconnector.system.tests.local;
1616

17+
import com.azure.storage.blob.BlobServiceClient;
18+
import com.github.javafaker.Faker;
1719
import org.eclipse.dataspaceconnector.azure.blob.core.AzureBlobStoreSchema;
20+
import org.eclipse.dataspaceconnector.azure.testfixtures.TestFunctions;
1821
import org.eclipse.dataspaceconnector.spi.types.TypeManager;
1922
import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress;
2023
import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferType;
@@ -24,6 +27,8 @@
2427
import java.time.Duration;
2528
import java.util.Map;
2629

30+
import static org.assertj.core.api.Assertions.assertThat;
31+
import static org.eclipse.dataspaceconnector.system.tests.utils.TransferSimulationUtils.PROVIDER_ASSET_FILE;
2732
import static org.eclipse.dataspaceconnector.system.tests.utils.TransferSimulationUtils.PROVIDER_ASSET_ID;
2833

2934
/**
@@ -32,11 +37,12 @@
3237
*/
3338
public class BlobTransferSimulationConfiguration implements TransferSimulationConfiguration {
3439

35-
private final String accountName;
36-
private final Integer maxSeconds;
40+
private final BlobServiceClient blobServiceClient;
41+
private final int maxSeconds;
42+
static final String BLOB_CONTENT = Faker.instance().lorem().sentence();
3743

38-
public BlobTransferSimulationConfiguration(String accountName, Integer maxSeconds) {
39-
this.accountName = accountName;
44+
public BlobTransferSimulationConfiguration(String accountName, String accountKey, String accountEndpoint, int maxSeconds) {
45+
this.blobServiceClient = TestFunctions.getBlobServiceClient(accountName, accountKey, accountEndpoint);
4046
this.maxSeconds = maxSeconds;
4147
}
4248

@@ -50,7 +56,7 @@ public String createTransferRequest(TransferInitiationData transferInitiationDat
5056
"protocol", "ids-multipart",
5157
"dataDestination", DataAddress.Builder.newInstance()
5258
.type(AzureBlobStoreSchema.TYPE)
53-
.property(AzureBlobStoreSchema.ACCOUNT_NAME, accountName)
59+
.property(AzureBlobStoreSchema.ACCOUNT_NAME, blobServiceClient.getAccountName())
5460
.build(),
5561
"managedResources", true,
5662
"transferType", TransferType.Builder.transferType()
@@ -62,8 +68,23 @@ public String createTransferRequest(TransferInitiationData transferInitiationDat
6268
return new TypeManager().writeValueAsString(request);
6369
}
6470

65-
@Override
6671
public Duration copyMaxDuration() {
6772
return Duration.ofSeconds(maxSeconds);
6873
}
74+
75+
@Override
76+
public boolean isTransferResultValid(Map<String, String> dataDestinationProperties) {
77+
// Assert
78+
var container = dataDestinationProperties.get("container");
79+
var destinationBlob = blobServiceClient.getBlobContainerClient(container)
80+
.getBlobClient(PROVIDER_ASSET_FILE);
81+
assertThat(destinationBlob.exists())
82+
.withFailMessage("Destination blob %s not created", destinationBlob.getBlobUrl())
83+
.isTrue();
84+
var actualBlobContent = destinationBlob.downloadContent().toString();
85+
assertThat(actualBlobContent)
86+
.withFailMessage("Transferred file contents are not same as the source file")
87+
.isEqualTo(BLOB_CONTENT);
88+
return true;
89+
}
6990
}

0 commit comments

Comments
 (0)