Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Emit ECS Meta metrics even when no prometheus instrumentation #309

Merged
merged 1 commit into from
Jul 17, 2023
+122 −92
Merged

Emit ECS Meta metrics even when no prometheus instrumentation #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions config/src/main/java/ai/asserts/aws/config/ScrapeConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,6 @@ public class ScrapeConfig {
@Builder.Default
private Map<String, SubnetDetails> primaryExporterByAccount = new TreeMap<>();

@Builder.Default
private Integer listMetricsResultCacheTTLMinutes = 10;

@Builder.Default
private Integer listFunctionsResultCacheTTLMinutes = 5;

@Builder.Default
private Integer getResourcesResultCacheTTLMinutes = 5;

@Builder.Default
private Integer numTaskThreads = 5;

@Builder.Default
private AuthConfig authConfig = new AuthConfig();

Expand All @@ -93,9 +81,6 @@ public class ScrapeConfig {
@Builder.Default
private Integer delay = 0;

@Builder.Default
private Integer logScrapeDelaySeconds = 15;

private TagExportConfig tagExportConfig;

private String alertForwardUrl;
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/ai/asserts/aws/exporter/ECSServiceDiscoveryExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ void writeFile(ScrapeConfig scrapeConfig, List<StaticConfig> targets, String fil

@VisibleForTesting
boolean shouldScrapeTargets(ScrapeConfig scrapeConfig, StaticConfig config) {
if( config.getTargets().isEmpty() ) {
return false;
}
String targetVpc = config.getLabels().getVpcId();
String targetSubnet = config.getLabels().getSubnetId();
boolean vpcOK = scrapeConfig.isDiscoverECSTasksAcrossVPCs() ||
Expand Down
145 changes: 78 additions & 67 deletions src/main/java/ai/asserts/aws/exporter/ECSTaskUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
*/
package ai.asserts.aws.exporter;

import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.AWSApiCallRateLimiter;
import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.TagUtil;
import ai.asserts.aws.TaskExecutorUtil;
import ai.asserts.aws.account.AWSAccount;
Expand Down Expand Up @@ -75,7 +75,8 @@ public class ECSTaskUtil {
public static final String PROMETHEUS_METRIC_PATH_DOCKER_LABEL = "PROMETHEUS_EXPORTER_PATH";


public ECSTaskUtil(AWSClientProvider awsClientProvider, ResourceMapper resourceMapper, AWSApiCallRateLimiter rateLimiter,
public ECSTaskUtil(AWSClientProvider awsClientProvider, ResourceMapper resourceMapper,
AWSApiCallRateLimiter rateLimiter,
TagUtil tagUtil, TaskExecutorUtil taskExecutorUtil) {
this.awsClientProvider = awsClientProvider;
this.resourceMapper = resourceMapper;
Expand All @@ -86,17 +87,6 @@ public ECSTaskUtil(AWSClientProvider awsClientProvider, ResourceMapper resourceM
envName = getInstallEnvName();
}

@VisibleForTesting
String getInstallEnvName() {
final String envName;
if (System.getenv(INSTALLED_ENV_NAME) != null) {
envName = System.getenv(INSTALLED_ENV_NAME);
} else {
envName = null;
}
return envName;
}

public boolean hasAllInfo(Task task) {
return "RUNNING".equals(task.lastStatus()) && task.hasAttachments() && task.attachments()
.stream()
Expand Down Expand Up @@ -142,33 +132,26 @@ public List<StaticConfig> buildScrapeTargets(AWSAccount account, ScrapeConfig sc
Optional<String> portFromLabel = getDockerLabel(cD, PROMETHEUS_PORT_DOCKER_LABEL);
labelsBuilder.availabilityZone(task.availabilityZone());
String jobName = cD.name();
labelsBuilder.container(jobName);
labelsBuilder.job(jobName);
if (pathFromLabel.isPresent() && portFromLabel.isPresent()) {
log.debug("Found prometheus port={}, path={} from docker labels for container {}/{}",
portFromLabel.get(),
pathFromLabel.get(),
taskDefinition.taskDefinitionArn(),
cD.name());
Labels labels = labelsBuilder
.job(jobName)
.metricsPath(pathFromLabel.get())
.container(cD.name())
.build();

labels.populateMapEntries();
labels.putAll(tagLabels);
StaticConfig staticConfig = targetsByLabel.computeIfAbsent(
labels, k -> StaticConfig.builder().labels(labels).build());
StaticConfig staticConfig = buildStaticConfig(tagLabels, targetsByLabel, cD, labels);
staticConfig.getTargets().add(format("%s:%s", ipAddress, portFromLabel.get()));
if (cD.logConfiguration() != null) {
LogConfig logConfig = LogConfig.builder()
.logDriver(cD.logConfiguration().logDriver().toString())
.options(cD.logConfiguration().options()).build();
staticConfig.getLogConfigs().add(logConfig);
}
} else {
log.warn("Docker labels for prometheus port and path not found for container {}/{}",
taskDefinition.taskDefinitionArn(),
cD.name());
Labels labels = labelsBuilder.build();
buildStaticConfig(tagLabels, targetsByLabel, cD, labels);
}
});
} else {
Expand All @@ -187,6 +170,58 @@ public List<StaticConfig> buildScrapeTargets(AWSAccount account, ScrapeConfig sc
return targets;
}

public SubnetDetails getSubnetDetails(Resource taskResource) {
EcsClient ecsClient = awsClientProvider.getECSClient(taskResource.getRegion(), AWSAccount.builder()
.accountId(taskResource.getAccount())
.build());
DescribeTasksResponse response = rateLimiter.doWithRateLimit("EcsClient/describeTasks",
ImmutableSortedMap.of(
SCRAPE_ACCOUNT_ID_LABEL, taskResource.getAccount(),
SCRAPE_REGION_LABEL, taskResource.getRegion(),
SCRAPE_OPERATION_LABEL, "EcsClient/describeTasks"),
() -> ecsClient.describeTasks(DescribeTasksRequest.builder()
.cluster(taskResource.getChildOf().getName())
.tasks(taskResource.getArn())
.build()));
if (response.hasTasks()) {
return response.tasks().get(0).attachments().stream()
.filter(attachment -> attachment.type().equals("ElasticNetworkInterface"))
.findFirst()
.flatMap(attachment -> attachment.details().stream()
.filter(kv -> kv.name().equals("subnetId")).findFirst())
.map(kv -> {
AtomicReference<String> vpcId = new AtomicReference<>("");
AtomicReference<String> subnetId = new AtomicReference<>("");
subnetId.set(kv.value());
vpcId.set(subnetIdMap.computeIfAbsent(subnetId.get(), kk ->
getVpcId(taskResource, subnetId)));
return SubnetDetails.builder()
.vpcId(vpcId.get())
.subnetId(subnetId.get())
.build();
}).orElse(null);
}
log.warn("Failed to find description for {}", taskResource);
return null;
}

@VisibleForTesting
String getEnv(AWSAccount account) {
return account.getName() != null ? account.getName() : envName != null ? envName : account.getAccountId();
}

@VisibleForTesting
String getInstallEnvName() {
final String envName;
if (System.getenv(INSTALLED_ENV_NAME) != null) {
envName = System.getenv(INSTALLED_ENV_NAME);
} else {
envName = null;
}
return envName;
}


@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private LabelsBuilder getLabelsBuilder(AWSAccount account, Resource cluster, Optional<String> service, Task task) {
Resource taskDefResource = resourceMapper.map(task.taskDefinitionArn())
Expand Down Expand Up @@ -232,8 +267,19 @@ private LabelsBuilder getLabelsBuilder(AWSAccount account, Resource cluster, Opt
return labelsBuilder;
}

private String getEnv(AWSAccount account) {
return envName != null ? envName : account.getName() != null ? account.getName() : account.getAccountId();
private StaticConfig buildStaticConfig(Map<String, String> tagLabels, Map<Labels, StaticConfig> targetsByLabel,
ContainerDefinition cD, Labels labels) {
labels.populateMapEntries();
labels.putAll(tagLabels);
StaticConfig staticConfig = targetsByLabel.computeIfAbsent(
labels, k -> StaticConfig.builder().labels(labels).build());
if (cD.logConfiguration() != null) {
LogConfig logConfig = LogConfig.builder()
.logDriver(cD.logConfiguration().logDriver().toString())
.options(cD.logConfiguration().options()).build();
staticConfig.getLogConfigs().add(logConfig);
}
return staticConfig;
}

private String getIPAddress(Task task) {
Expand All @@ -249,39 +295,11 @@ private String getIPAddress(Task task) {
return ipAddress;
}

public SubnetDetails getSubnetDetails(Resource taskResource) {
EcsClient ecsClient = awsClientProvider.getECSClient(taskResource.getRegion(), AWSAccount.builder()
.accountId(taskResource.getAccount())
.build());
DescribeTasksResponse response = rateLimiter.doWithRateLimit("EcsClient/describeTasks",
ImmutableSortedMap.of(
SCRAPE_ACCOUNT_ID_LABEL, taskResource.getAccount(),
SCRAPE_REGION_LABEL, taskResource.getRegion(),
SCRAPE_OPERATION_LABEL, "EcsClient/describeTasks"),
() -> ecsClient.describeTasks(DescribeTasksRequest.builder()
.cluster(taskResource.getChildOf().getName())
.tasks(taskResource.getArn())
.build()));
if (response.hasTasks()) {
return response.tasks().get(0).attachments().stream()
.filter(attachment -> attachment.type().equals("ElasticNetworkInterface"))
.findFirst()
.flatMap(attachment -> attachment.details().stream()
.filter(kv -> kv.name().equals("subnetId")).findFirst())
.map(kv -> {
AtomicReference<String> vpcId = new AtomicReference<>("");
AtomicReference<String> subnetId = new AtomicReference<>("");
subnetId.set(kv.value());
vpcId.set(subnetIdMap.computeIfAbsent(subnetId.get(), kk ->
getVpcId(taskResource, subnetId)));
return SubnetDetails.builder()
.vpcId(vpcId.get())
.subnetId(subnetId.get())
.build();
}).orElse(null);
}
log.warn("Failed to find description for {}", taskResource);
return null;
private Optional<String> getDockerLabel(ContainerDefinition container, String labelName) {
return container.dockerLabels().entrySet().stream()
.filter(entry -> entry.getKey().equals(labelName))
.map(Map.Entry::getValue)
.findFirst();
}

private SubnetDetails getSubnetDetails(Task task, Resource taskResource) {
Expand Down Expand Up @@ -321,11 +339,4 @@ private String getVpcId(Resource taskResource, AtomicReference<String> subnetId)
r.subnets().stream().findFirst().ifPresent(subnet -> id.set(subnet.vpcId()));
return id.get();
}

Optional<String> getDockerLabel(ContainerDefinition container, String labelName) {
return container.dockerLabels().entrySet().stream()
.filter(entry -> entry.getKey().equals(labelName))
.map(Map.Entry::getValue)
.findFirst();
}
}
31 changes: 23 additions & 8 deletions src/test/java/ai/asserts/aws/exporter/ECSServiceDiscoveryExporterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.easymock.EasyMockSupport;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -238,36 +239,49 @@ void identifySubnetsToScrape() {
.vpcId("vpc-1")
.subnetId("subnet-1")
.build());
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
StaticConfig staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-1")
.subnetId("subnet-1")
.build())
.build()));
.build();

// Without target
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// With target
staticConfig.getTargets().add("1.2.3.4:8080");
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// Same VPC, different subnet. But subnet configured to be scraped
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-1")
.subnetId("subnet-2")
.build())
.build()));
.build();
staticConfig.getTargets().add("1.2.3.4:8080");
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// Same VPC, different subnet. But subnet not configured to be scraped
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-1")
.subnetId("subnet-3")
.build())
.build()));
.build();
staticConfig.getTargets().add("1.2.3.4:8080");
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// Different VPC
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-2")
.subnetId("subnet-1")
.build())
.build()));
.build();
staticConfig.getTargets().add("1.2.3.4:8080");
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));
verifyAll();
}

Expand All @@ -283,6 +297,7 @@ public void run() throws Exception {
expect(ecsTaskProvider.getScrapeTargets()).andReturn(ImmutableList.of(mockStaticConfig, mockStaticConfig));

expect(scrapeConfig.isLogECSTargets()).andReturn(true);
expect(mockStaticConfig.getTargets()).andReturn(ImmutableSet.of("1.2.3.4:8080")).anyTimes();
expect(mockStaticConfig.getLabels()).andReturn(mockLabels).anyTimes();
expect(mockLabels.getVpcId()).andReturn("vpc-id").anyTimes();
expect(mockLabels.getSubnetId()).andReturn("subnet-id").anyTimes();
Expand Down
20 changes: 18 additions & 2 deletions src/test/java/ai/asserts/aws/exporter/ECSTaskUtilTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
*/
package ai.asserts.aws.exporter;

import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.AWSApiCallRateLimiter;
import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.TagUtil;
import ai.asserts.aws.TaskExecutorUtil;
import ai.asserts.aws.TestTaskThreadPool;
Expand Down Expand Up @@ -66,6 +66,7 @@ public class ECSTaskUtilTest extends EasyMockSupport {
private TagUtil tagUtil;
private ScrapeConfig scrapeConfig;
private AWSAccount account;
private String defaultEnvName;

@BeforeEach
public void setup() {
Expand All @@ -85,9 +86,15 @@ public void setup() {
Ec2Client ec2Client = mock(Ec2Client.class);


defaultEnvName = "dev";
testClass = new ECSTaskUtil(awsClientProvider, resourceMapper,
rateLimiter, tagUtil,
taskExecutorUtil);
taskExecutorUtil) {
@Override
String getInstallEnvName() {
return defaultEnvName;
}
};

expect(awsClientProvider.getEc2Client(anyString(), anyObject())).andReturn(ec2Client).anyTimes();
expect(ec2Client.describeSubnets(DescribeSubnetsRequest.builder()
Expand Down Expand Up @@ -144,6 +151,15 @@ public void hasAllInfo_true() {
verifyAll();
}

@Test
public void getEnvName() {
assertEquals("prod", testClass.getEnv(AWSAccount.builder()
.name("prod")
.build()));
assertEquals(defaultEnvName, testClass.getEnv(AWSAccount.builder()
.build()));
}

@Test
public void containerWithDockerLabels() {
expect(resourceMapper.map("task-def-arn")).andReturn(Optional.of(taskDef));
Expand Down
Loading