import os import click from sdi_checker.app_cli import display_check @click.command(name='csw') @click.argument('url', nargs=-1) # help = the server to target (full URL, e.g. https://sdi.georchestra.org/geoserver/wms) @click.option('--inspire', '-i', show_default=True, default='flexible', type=click.Choice(['flexible', 'strict'], case_sensitive=False), help='indicates if the checks should be strict or flexible, default to flexible') @click.option('--geoserver-to-check', '-g', default=None, type=str, multiple=True, help="space-separated list of geoserver hostname to check in CSW mode with inspire strict option activated. Ex: sdi.georchestra.org") @click.option('--disable-ssl-verification', '-ssl', is_flag=True, type=bool, help="disable certificate verification") @click.option('--only-err', '-e', is_flag=True, type=bool, help="only display errors, no summary informations will be displayed") @click.option('--xunit', '-x', is_flag=True, type=bool, help="generate a XML xunit result report") @click.option('--xunit-output', '-xo', default="xunit.xml", show_default=True, type=str, help="name of the xunit report file, defaults to ./xunit.xml") @click.option('--log-to-file', '-l', default='', type=str, help="if a file path is specified, log output to this file, not stdout") @click.option('--timeout', '-t', default=0, type=int, help="specify a timeout for request to external service.") @click.option('--output', '-o', type=click.Choice(['json', 'txt', 'screen', 'csv', 'db'], case_sensitive=False), default='screen', help="output format") @click.pass_obj def run(app, url, inspire, geoserver_to_check, disable_ssl_verification, only_err, xunit, xunit_output, log_to_file, timeout, output): """ > master add [TEXT] [--template TPL] [--file FILE] [--edit] [--repository REPOSITORY] [--login LOGIN] [--password PASSWORD] """ on_csw(app, url, inspire, geoserver_to_check, disable_ssl_verification, only_err, xunit, xunit_output, log_to_file, timeout, output) def on_csw(app, url, inspire, geoserver_to_check, disable_ssl_verification, only_err, xunit, xunit_output, log_to_file, timeout, output): """ > master add [TEXT] [--template TPL] [--file FILE] [--edit] (=> open edit window) """ # if helpers.check_repository_connect(app, repository, login, password): # # Backup main directory # app.backup_files() # text = '\n====\n'.join(text) if text else None # # TODO: check use of --edit, --file and --template parameters # if text is None or edit or file or template: # path = None # if file: # path = os.path.split(file) # if template: # path = [app.templates_directory, template + '.txt'] # text = app.get_template(path[1], path[0]) if path is not None else text # text = click.edit(text) or None # if text is not None: # insert_result = app.add_items(text) # save_result = app.save_items_to_files(insert_result['items']) # # Display clean report # display.display_save_report(app, save_result['total_items'], save_result['total_bad_items'], save_result['total_good_items'], save_result['total_wrote_items']) # app.echo() # app.echo('{nb_added_items} items saved (total: {nb_total_new})'.format(nb_added_items=insert_result['nb_added_items'], nb_total_new=insert_result['nb_total_new'])) # app.echo('Added items MD5: {md5}'.format(md5=insert_result['md5'])) # if insert_result['nb_exist_items'] > 0: # app.echo('{nb_exist_items} item(s) already exist(s)'.format(nb_exist_items=insert_result['nb_exist_items'])) # app.echo() # app.rprompt = '{nb_added_items} item(s) saved'.format(nb_added_items=insert_result['nb_added_items']) # else: # app.echo('No item added.') # app.echo() if url is not None: total_mds = 0 geoserver_services = CachedOwsServices(creds, disable_ssl=args.disable_ssl_verification, timeout=request_timeout) try: csw_q = CSWQuerier(args.server, credentials=creds, cached_ows_services=geoserver_services, logger=logger, timeout=request_timeout) except ServiceException as e: logger.debug(e, exc_info=True) logger.fatal("Unable to query the remote CSW:\nError: %s\nPlease check the CSW url", e) sys.exit(1) errors = [] reporting = [] if args.inspire == "strict": # Step 1: get all data metadata datamd = csw_q.get_all_records(constraints=[And([csw_q.is_dataset, csw_q.non_harvested])]) # Step 2: maps data metadatas to service MDs servicesmd = csw_q.get_all_records(constraints=[And([csw_q.is_service, csw_q.non_harvested])]) data_to_service_map = {} for uuid, md in servicesmd.items(): for oon in md.identificationinfo[0].operateson: if data_to_service_map.get(oon['uuidref']) is None: data_to_service_map[oon['uuidref']] = [uuid] else: data_to_service_map[oon['uuidref']] = data_to_service_map[oon['uuidref']] + [uuid] # Step 3: on each data md, get the service md, and the underlying service URL #for uuid, md in enumerate(datamd): for mdd_uuid, mdd in datamd.items(): # Note: this won't count the service metadata in the end, only the MDD that trigger a # check onto a service MD. total_mds += 1 if data_to_service_map.get(mdd_uuid) is None: # TODO file an issue if the dataMd has no ServiceMd linked to ? if len([x for x in reporting if x['uuid'] == mdd_uuid]) == 0: reporting.append({ 'classname': 'CSW', 'name': mdd.identification.title, 'uuid': mdd_uuid, 'time': '0', 'error': None }) continue # step 4: check the layer existence using the service URL for sce_uuid in data_to_service_map[mdd_uuid]: try: mds = servicesmd[sce_uuid] mdd = datamd[mdd_uuid] csw_q.check_service_md(mds, mdd, geoserver_to_check=args.geoserver_to_check if args.geoserver_to_check is not None else []) # No issue so far ? # since a MDD can reference several service metadata, consider # the MDD as passing tests only once (avoid adding several times the same MDD # to the array). It must be very unlikely to have several MDS anyway. if len([x for x in reporting if x['uuid'] == mdd_uuid]) == 0: reporting.append({ 'classname': 'CSW', 'name': mdd.title, 'uuid': mdd_uuid, 'time': '0', 'error': None }) except Inconsistency as e: logger.debug(e, exc_info=True) logger.error(e) errors.append(e) # Same as above: only adding the errored MDD once if len([x for x in reporting if x['uuid'] == mdd_uuid]) == 0: reporting.append({ 'classname': 'CSW', 'name': mdd.title, 'uuid': mdd_uuid, 'time': '0', 'error': e }) elif args.inspire == "flexible": global_idx = 0 csw_q.start = 0 while True: res = csw_q.get_dataset_records(constraints=[csw_q.non_harvested]) total_mds += len(res) for idx, uuid in enumerate(res): current_md = res[uuid] logger.info("#%d\n UUID : %s\n %s", global_idx, uuid, current_md.title) wms_found = False wfs_found = False for uri in csw_q.get_md(uuid).uris: from_wms = False try: if uri["protocol"] == "OGC:WMS": wms_found = True from_wms = True # TODO: use the geoserver_to_check option ? geoserver_services.checkWmsLayer(uri["url"], uri["name"]) logger.debug("\tURI OK : %s %s %s", uri["protocol"], uri['url'], uri['name']) logger.info(" WMS url: OK") elif uri["protocol"] == "OGC:WFS": wfs_found = True # TODO: same remark geoserver_services.checkWfsLayer(uri["url"], uri["name"]) logger.debug("\tURI OK : %s %s %s", uri["protocol"], uri['url'], uri['name']) logger.info(" WFS url: OK") else: logger.debug("\tSkipping URI : %s %s %s", uri["protocol"], uri['url'], uri['name']) except Exception as ex: if isinstance(ex, GnToGsLayerNotFoundInconsistency) or \ isinstance(ex, GnToGsInvalidCapabilitiesUrl) or \ isinstance(ex,GnToGsOtherError): ex.set_md_uuid(uuid) errors.append(ex) else: # morph encountered error in to an "other error" exc = GnToGsOtherError(uri['url'], uri['name'], ex) exc.set_md_uuid(uuid) errors.append(exc) logger.debug("\t /!\\ ---> Cannot find Layer ON GS : %s %s %s %s %s", uuid, uri['protocol'], uri['url'], uri['name'], ex) logger.info(" %s url: KO: %s: %s" % ("WMS" if from_wms else "WFS", uri['url'], str(errors[-1]))) # in both cases, add the MDD in the reporting array if len([x for x in reporting if x['uuid'] == uuid]) == 0: reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid, 'time': '0', 'error': ex }) if not wms_found: logger.info(" WMS url: KO: No wms url found in the metadata") err = GnToGsNoOGCWmsDefined(uuid) errors.append(err) reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid, 'time': '0', 'error': err }) if not wfs_found: logger.info(" WFS url: KO: No wfs url found in the metadata") err = GnToGsNoOGCWfsDefined(uuid) errors.append(err) reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid, 'time': '0', 'error': err }) if wms_found and wfs_found: reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid, 'time': '0', 'error': None }) logger.info("") # end of current md global_idx += 1 if csw_q.start > csw_q.csw.results['matches']: break print_csw_report(errors, total_mds) if args.xunit: generate_csw_xunit_layers_status(reporting, args.xunit_output)