diff --git a/.mega-linter.yml b/.mega-linter.yml index a3e1c30e..fd35ce64 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -15,6 +15,7 @@ DISABLE: DISABLE_LINTERS: - YAML_V8R PRINT_ALL_FILES: false +BASH_SHFMT_ARGUMENTS: "-i 4" DOCKERFILE_HADOLINT_ARGUMENTS: "--ignore DL3008 --ignore DL4006 --ignore DL3009 --ignore DL3059" SHOW_ELAPSED_TIME: true FLAVOR_SUGGESTIONS: false diff --git a/import/import_from_ckan.sh b/import/import_from_ckan.sh index 101d868c..5a56cba7 100755 --- a/import/import_from_ckan.sh +++ b/import/import_from_ckan.sh @@ -2,30 +2,28 @@ function usage() { echo -n \ - "Usage: $(basename "$0") ckan_site_to_import + "Usage: $(basename "$0") ckan_site_to_import Imports data from a CKAN site. Requires one positional argument giving the site to import from. " } -if [ "${1:-}" = "--help" ] || [[ $# -eq 0 ]] -then +if [ "${1:-}" = "--help" ] || [[ $# -eq 0 ]]; then usage else - if [ ! -d working_files ] # only generates working files if the dir doesn't exist - then + if [ ! -d working_files ]; then # only generates working files if the dir doesn't exist echo "Seems like you need working files. Generating them now!" mkdir working_files # Get dataset names from ckan_site_to_import - curl -k "${1}" | jq -r '.result | .[]' > working_files/all_dataset_names.txt + curl -k "${1}" | jq -r '.result | .[]' >working_files/all_dataset_names.txt # Get the datasets and filter out failed requests # shellcheck disable=SC2002 - cat working_files/all_dataset_names.txt | xargs -d"\n" --replace='{}' curl -k -fw "\n" 'https://ckan.opendataphilly.org/api/action/package_show?id={}&include_datasets=False&include_password_hash=True' > working_files/odp_datasets.txt + cat working_files/all_dataset_names.txt | xargs -d"\n" --replace='{}' curl -k -fw "\n" 'https://ckan.opendataphilly.org/api/action/package_show?id={}&include_datasets=False&include_password_hash=True' >working_files/odp_datasets.txt # Convert JSONL/JSONND to JSON # shellcheck disable=SC2002 - cat working_files/odp_datasets.txt | jq -s > working_files/odp_datasets.json + cat working_files/odp_datasets.txt | jq -s >working_files/odp_datasets.json else echo "Seems like you already have working files, so I'll try to use those! If you don't, please delete the /working_files directory and I'll regenerate them."