Merge pull request #592 from sromic/main

CHANGE - fixed not getting all paginated results from AWS clients calls

src/aws/acm-wrapper.ts

@@ -1,10 +1,13 @@
 import {
-    ACMClient,
-    ListCertificatesCommand,
-    ListCertificatesCommandOutput,
+  ACMClient,
+  CertificateSummary,
+  ListCertificatesCommand,
+  ListCertificatesCommandInput,
+  ListCertificatesCommandOutput
 } from "@aws-sdk/client-acm";
 import Globals from "../globals";
 import DomainConfig = require("../models/domain-config");
+import { getAWSPagedResults } from "../utils";
 
 const certStatuses = ["PENDING_VALIDATION", "ISSUED", "INACTIVE"];
 
@@ -15,7 +18,8 @@ class ACMWrapper {
         const isEdge = endpointType === Globals.endpointTypes.edge;
         this.acm = new ACMClient({
             credentials,
-            region: isEdge ? Globals.defaultRegion : Globals.getRegion()
+            region: isEdge ? Globals.defaultRegion : Globals.getRegion(),
+            retryStrategy: Globals.getRetryStrategy()
         });
     }
 
@@ -24,16 +28,20 @@ class ACMWrapper {
         let certificateName = domain.certificateName; // The certificate name
 
         try {
-            const response: ListCertificatesCommandOutput = await this.acm.send(
-                new ListCertificatesCommand({CertificateStatuses: certStatuses})
+            const certificates = await getAWSPagedResults<CertificateSummary, ListCertificatesCommandInput, ListCertificatesCommandOutput>(
+              this.acm,
+              "CertificateSummaryList",
+              "NextToken",
+              "NextToken",
+              new ListCertificatesCommand({ CertificateStatuses: certStatuses })
             );
             // enhancement idea: weight the choice of cert so longer expires
             // and RenewalEligibility = ELIGIBLE is more preferable
             if (certificateName) {
-                certificateArn = this.getCertArnByCertName(response.CertificateSummaryList, certificateName);
+                certificateArn = this.getCertArnByCertName(certificates, certificateName);
             } else {
                 certificateName = domain.givenDomainName;
-                certificateArn = this.getCertArnByDomainName(response.CertificateSummaryList, certificateName);
+                certificateArn = this.getCertArnByDomainName(certificates, certificateName);
             }
         } catch (err) {
             throw Error(`Could not search certificates in Certificate Manager.\n${err.message}`);

src/aws/api-gateway-v1-wrapper.ts

@@ -5,24 +5,27 @@ import DomainConfig = require("../models/domain-config");
 import DomainInfo = require("../models/domain-info");
 import Globals from "../globals";
 import {
-    APIGatewayClient,
-    CreateBasePathMappingCommand,
-    CreateDomainNameCommand,
-    CreateDomainNameCommandOutput,
-    DeleteBasePathMappingCommand,
-    DeleteDomainNameCommand,
-    GetBasePathMappingsCommand,
-    GetBasePathMappingsCommandOutput,
-    GetDomainNameCommand,
-    GetDomainNameCommandOutput,
-    UpdateBasePathMappingCommand,
+  APIGatewayClient,
+  BasePathMapping,
+  CreateBasePathMappingCommand,
+  CreateDomainNameCommand,
+  CreateDomainNameCommandOutput,
+  DeleteBasePathMappingCommand,
+  DeleteDomainNameCommand,
+  GetBasePathMappingsCommand,
+  GetBasePathMappingsCommandInput,
+  GetBasePathMappingsCommandOutput,
+  GetDomainNameCommand,
+  GetDomainNameCommandOutput,
+  UpdateBasePathMappingCommand
 } from "@aws-sdk/client-api-gateway";
 import ApiGatewayMap = require("../models/api-gateway-map");
 import APIGatewayBase = require("../models/apigateway-base");
 import Logging from "../logging";
+import { getAWSPagedResults } from "../utils";
 
 class APIGatewayV1Wrapper extends APIGatewayBase {
-    constructor(credentials?: any,) {
+    constructor(credentials?: any) {
         super();
         this.apiGateway = new APIGatewayClient({
             credentials,
@@ -124,12 +127,15 @@ class APIGatewayV1Wrapper extends APIGatewayBase {
 
     public async getBasePathMappings(domain: DomainConfig): Promise<ApiGatewayMap[]> {
         try {
-            const response: GetBasePathMappingsCommandOutput = await this.apiGateway.send(
+            const items = await getAWSPagedResults<BasePathMapping, GetBasePathMappingsCommandInput, GetBasePathMappingsCommandOutput>(
+                this.apiGateway,
+                "items",
+                "position",
+                "position",
                 new GetBasePathMappingsCommand({
-                    domainName: domain.givenDomainName
+                    domainName: domain.givenDomainName,
                 })
             );
-            const items = response.items || [];
             return items.map((item) => {
                     return new ApiGatewayMap(item.restApiId, item.basePath, item.stage, null);
                 }

src/aws/api-gateway-v2-wrapper.ts

@@ -7,22 +7,24 @@ import Globals from "../globals";
 import ApiGatewayMap = require("../models/api-gateway-map");
 import APIGatewayBase = require("../models/apigateway-base");
 import {
-    ApiGatewayV2Client,
-    CreateApiMappingCommand,
-    CreateDomainNameCommand,
-    CreateDomainNameCommandOutput,
-    DeleteApiMappingCommand,
-    DeleteDomainNameCommand,
-    GetApiMappingsCommand,
-    GetApiMappingsCommandOutput,
-    GetDomainNameCommand,
-    GetDomainNameCommandOutput,
-    UpdateApiMappingCommand,
+  ApiGatewayV2Client,
+  ApiMapping,
+  CreateApiMappingCommand,
+  CreateDomainNameCommand,
+  CreateDomainNameCommandOutput,
+  DeleteApiMappingCommand,
+  DeleteDomainNameCommand,
+  GetApiMappingsCommand,
+  GetApiMappingsCommandInput,
+  GetApiMappingsCommandOutput,
+  GetDomainNameCommand,
+  GetDomainNameCommandOutput,
+  UpdateApiMappingCommand
 } from "@aws-sdk/client-apigatewayv2";
 import Logging from "../logging";
+import { getAWSPagedResults } from "../utils";
 
 class APIGatewayV2Wrapper extends APIGatewayBase {
-
     constructor(credentials?: any) {
         super();
         this.apiGateway = new ApiGatewayV2Client({
@@ -153,12 +155,15 @@ class APIGatewayV2Wrapper extends APIGatewayBase {
      */
     public async getBasePathMappings(domain: DomainConfig): Promise<ApiGatewayMap[]> {
         try {
-            const response: GetApiMappingsCommandOutput = await this.apiGateway.send(
-                new GetApiMappingsCommand({
-                    DomainName: domain.givenDomainName
-                })
+            const items = await getAWSPagedResults<ApiMapping, GetApiMappingsCommandInput, GetApiMappingsCommandOutput>(
+              this.apiGateway,
+              "Items",
+              "NextToken",
+              "NextToken",
+              new GetApiMappingsCommand({
+                DomainName: domain.givenDomainName
+              })
             );
-            const items = response.Items || [];
             return items.map(
                 (item) => new ApiGatewayMap(item.ApiId, item.ApiMappingKey, item.Stage, item.ApiMappingId)
             );

src/aws/cloud-formation-wrapper.ts

@@ -2,17 +2,22 @@
  * Wrapper class for AWS CloudFormation provider
  */
 
-import Globals from "../globals";
-import Logging from "../logging";
 import {
-    CloudFormationClient,
-    DescribeStackResourceCommand,
-    DescribeStackResourceCommandOutput,
-    DescribeStacksCommand,
-    DescribeStacksCommandOutput,
-    ListExportsCommand,
-    ListExportsCommandOutput
+  CloudFormationClient,
+  DescribeStackResourceCommand,
+  DescribeStackResourceCommandOutput,
+  DescribeStacksCommand,
+  DescribeStacksCommandInput,
+  DescribeStacksCommandOutput,
+  Export,
+  ListExportsCommand,
+  ListExportsCommandInput,
+  ListExportsCommandOutput,
+  Stack
 } from "@aws-sdk/client-cloudformation";
+import Globals from "../globals";
+import Logging from "../logging";
+import { getAWSPagedResults } from "../utils";
 
 class CloudFormationWrapper {
     public cloudFormation: CloudFormationClient;
@@ -24,7 +29,8 @@ class CloudFormationWrapper {
         this.stackName = Globals.serverless.service.provider.stackName || defaultStackName;
         this.cloudFormation = new CloudFormationClient({
             credentials,
-            region: Globals.getRegion()
+            region: Globals.getRegion(),
+            retryStrategy: Globals.getRetryStrategy()
         });
     }
 
@@ -120,10 +126,13 @@ class CloudFormationWrapper {
      * Gets values by names from cloudformation exports
      */
     public async getImportValues(names: string[]): Promise<any> {
-        const response: ListExportsCommandOutput = await this.cloudFormation.send(
-            new ListExportsCommand({})
+        const exports = await getAWSPagedResults<Export, ListExportsCommandInput, ListExportsCommandOutput>(
+          this.cloudFormation,
+          "Exports",
+          "NextToken",
+          "NextToken",
+          new ListExportsCommand({})
         );
-        const exports = response.Exports || [];
         // filter Exports by names which we need
         const filteredExports = exports.filter((item) => names.indexOf(item.Name) !== -1);
         // converting a list of unique values to dict
@@ -152,10 +161,13 @@ class CloudFormationWrapper {
      */
     public async getNestedStack(logicalResourceId: string, stackName: string) {
         // get all stacks from the CloudFormation
-        const response: DescribeStacksCommandOutput = await this.cloudFormation.send(
-            new DescribeStacksCommand({})
+        const stacks = await getAWSPagedResults<Stack, DescribeStacksCommandInput, DescribeStacksCommandOutput>(
+          this.cloudFormation,
+          "Stacks",
+          "NextToken",
+          "NextToken",
+          new DescribeStacksCommand({})
         );
-        const stacks = response.Stacks || [];
 
         // filter stacks by given stackName and check by nested stack RootId
         const regex = new RegExp("/" + stackName + "/");

src/aws/route53-wrapper.ts

@@ -2,11 +2,14 @@ import Globals from "../globals";
 import DomainConfig = require("../models/domain-config");
 import Logging from "../logging";
 import {
-    ChangeResourceRecordSetsCommand,
-    ListHostedZonesCommand,
-    ListHostedZonesCommandOutput,
-    Route53Client
+  ChangeResourceRecordSetsCommand,
+  HostedZone,
+  ListHostedZonesCommand,
+  ListHostedZonesCommandInput,
+  ListHostedZonesCommandOutput,
+  Route53Client
 } from "@aws-sdk/client-route-53";
+import { getAWSPagedResults } from "../utils";
 
 class Route53Wrapper {
     public route53: Route53Client;
@@ -16,10 +19,14 @@ class Route53Wrapper {
         if (credentials) {
             this.route53 = new Route53Client({
                 credentials,
-                region: region || Globals.getRegion()
+                region: region || Globals.getRegion(),
+                retryStrategy: Globals.getRetryStrategy()
             });
         } else {
-            this.route53 = new Route53Client({region: Globals.getRegion()});
+            this.route53 = new Route53Client({
+                region: Globals.getRegion(),
+                retryStrategy: Globals.getRetryStrategy()
+            });
         }
     }
 
@@ -40,10 +47,13 @@ class Route53Wrapper {
 
         let hostedZones = [];
         try {
-            const response: ListHostedZonesCommandOutput = await this.route53.send(
-                new ListHostedZonesCommand({})
+            hostedZones = await getAWSPagedResults<HostedZone, ListHostedZonesCommandInput, ListHostedZonesCommandOutput>(
+              this.route53,
+              "HostedZones",
+              "Marker",
+              "NextMarker",
+              new ListHostedZonesCommand({})
             );
-            hostedZones = response.HostedZones || hostedZones;
         } catch (err) {
             throw new Error(`Unable to list hosted zones in Route53.\n${err.message}`);
         }

src/utils.ts

@@ -1,11 +1,13 @@
+import { Client, Command } from "@smithy/smithy-client";
+import { MetadataBearer } from "@smithy/types";
 import Globals from "./globals";
 
 /**
  * Stops event thread execution for given number of seconds.
  * @param seconds
  * @returns {Promise<void>} Resolves after given number of seconds.
  */
-async function sleep(seconds) {
+async function sleep(seconds: number) {
     return new Promise((resolve) => setTimeout(resolve, 1000 * seconds));
 }
 
@@ -37,7 +39,34 @@ function evaluateBoolean(value: any, defaultValue: boolean): boolean {
     throw new Error(`${Globals.pluginName}: Ambiguous boolean config: "${value}"`);
 }
 
-export {
-    evaluateBoolean,
-    sleep,
-};
+/**
+ * Iterate through the pages of a AWS SDK response and collect them into a single array
+ *
+ * @param client - The AWS service instance to use to make the calls
+ * @param resultsKey - The key name in the response that contains the items to return
+ * @param nextTokenKey - The request key name to append to the request that has the paging token value
+ * @param nextRequestTokenKey - The response key name that has the next paging token value
+ * @param params - Parameters to send in the request
+ */
+async function getAWSPagedResults<ClientOutput, ClientInputCommand extends object, ClientOutputCommand extends MetadataBearer>(
+  client: Client<any, any, any, any>,
+  resultsKey: keyof ClientOutputCommand,
+  nextTokenKey: keyof ClientInputCommand,
+  nextRequestTokenKey: keyof ClientOutputCommand,
+  params: Command<any, any, any>
+): Promise<ClientOutput[]> {
+  let results = [];
+  let response = await client.send(params);
+  results = results.concat(response[resultsKey] || results);
+  while (
+    response.hasOwnProperty(nextRequestTokenKey) &&
+    response[nextRequestTokenKey]
+  ) {
+    params.input[nextTokenKey] = response[nextRequestTokenKey];
+    response = await client.send(params);
+    results = results.concat(response[resultsKey]);
+  }
+  return results;
+}
+
+export { evaluateBoolean, sleep, getAWSPagedResults };