<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;"># flake8: noqa: E501


import os
import subprocess

import requests
from mm_stats.config import IGNITE_CONFIG_FILE, JAVA_PATH
from mm_stats.definitions import TEMP_DATA_PATH, logger


def check_io(process):
    """Check logs of java process and log to python logger."""
    while True:
        output = process.stdout.readline().decode()
        if output:
            logger.info(output)
        else:
            break


def download_oshdb_ignite_config_file():
    """Write oshdb ignite cluster config file and save to files."""
    ignite_client_config = """&lt;?xml version="1.0" encoding="UTF-8"?&gt;
    &lt;beans xmlns="http://www.springframework.org/schema/beans"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="
           http://www.springframework.org/schema/beans
           http://www.springframework.org/schema/beans/spring-beans.xsd"&gt;

        &lt;bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"&gt;
            &lt;!-- a name to identify you or your client --&gt;
            &lt;property name="igniteInstanceName" value="humstats_processing" /&gt;
            &lt;!-- the directory for temporary ignite files on your client (can be removed) --&gt;
            &lt;property name="workDirectory" value="/tmp/ignite"/&gt;
            &lt;!-- Explicitly enable peer class loading. --&gt;
            &lt;property name="peerClassLoadingEnabled" value="true"/&gt;
            &lt;property name="clientMode" value="true" /&gt;
            &lt;property name="discoverySpi"&gt;
                &lt;bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"&gt;
                    &lt;property name="ipFinder"&gt;
                        &lt;bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"&gt;
                            &lt;property name="addresses"&gt;
                                &lt;list&gt;
                                    &lt;value&gt;heigitsv01.heigit.org:47500&lt;/value&gt;
                                    &lt;value&gt;heigitsv02.heigit.org:47500&lt;/value&gt;
                                    &lt;value&gt;heigitsv07.heigit.org:47500&lt;/value&gt;
                                &lt;/list&gt;
                            &lt;/property&gt;
                        &lt;/bean&gt;
                    &lt;/property&gt;
                &lt;/bean&gt;
            &lt;/property&gt;
        &lt;/bean&gt;
    &lt;/beans&gt;
    """

    with open(IGNITE_CONFIG_FILE, "w") as f:
        f.write(ignite_client_config)
    logger.info(f"Created ignite config file and saved to: {IGNITE_CONFIG_FILE}")


def get_oshdb_ignite_prefix() -&gt; int:
    """Get ignite prefix for oshdb from ohsome api."""
    url = "https://api.ohsome.org/v1/metadata"
    r = requests.get(url)
    ignite_prefix = int(r.json()["extractRegion"]["replicationSequenceNumber"])
    logger.info(f"Downloaded ignite prefix: {ignite_prefix}")
    return ignite_prefix


def run_oshdb_query_users_per_country(
    country_id,
    timestamp_start,
    timestamp_end,
    cluster=True,
    oshdb_logs=False,
    interval="monthly",
):
    """Query OSM stats from oshdb for 'users' using maven commands."""
    main_class = "org.heigit.ohsome.ProjectAggregation.ProjectAggregationMain"
    cwd = os.path.join(JAVA_PATH, "ProjectAggregation")

    # use the oshdb ignite cluster defined in environment variables
    ignite_prefix = f"{get_oshdb_ignite_prefix()}"
    download_oshdb_ignite_config_file()

    if cluster:
        # use the oshdb ignite cluster defined in environment variables
        # download_oshdb_ignite_config_file()
        cluster_or_local_arg = "-c"
    else:
        # use the local oshdb file defined in environment variables
        ignite_prefix_arg = "-ignite=0"  # not needed for local setup
        cluster_or_local_arg = None  # no flat needed for local setup

    query = f"SELECT ST_asText(geom) AS WKT FROM data_preparation.ne_10m_admin_0_countries WHERE id = {country_id};"

    command = f"""
        export MAVEN_OPTS="-Xms4G -Xmx32G --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED --add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.sql/java.sql=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED"
        mvn compile exec:java\
        -Dexec.mainClass="{main_class}"\
        -Dexec.args="'{query}' {timestamp_start} {timestamp_end} {interval} {TEMP_DATA_PATH} {ignite_prefix} {country_id}"
        """

    logger.info(command)

    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, cwd=cwd)
    # keep checking stdout/stderr until the child exits
    if oshdb_logs:
        while process.poll() is None:
            check_io(process)

    process.stdout.close()
    process.wait()
    logger.info("finished oshdb query for users")
    logger.info(f"return code: {process.returncode} for country {country_id}")


def run_oshdb_query_users_per_project(
    project_id,
    timestamp_start,
    timestamp_end,
    cluster=True,
    oshdb_logs=False,
    interval="daily",
):
    """Query OSM stats from oshdb for 'users' using maven commands."""
    main_class = "org.heigit.ohsome.ProjectAggregation.ProjectAggregationMain"
    cwd = os.path.join(JAVA_PATH, "ProjectAggregation")

    # use the oshdb ignite cluster defined in environment variables
    ignite_prefix = f"{get_oshdb_ignite_prefix()}"
    download_oshdb_ignite_config_file()

    if cluster:
        # use the oshdb ignite cluster defined in environment variables
        # download_oshdb_ignite_config_file()
        cluster_or_local_arg = "-c"
    else:
        # use the local oshdb file defined in environment variables
        ignite_prefix_arg = "-ignite=0"  # not needed for local setup
        cluster_or_local_arg = None  # no flat needed for local setup

    query = f"""
        select
          case
            -- valid geometries, decent number of vertices
            -- still we run make valid, just to be on the safe side
            -- invalid geometry, but should be easy to fix as not so many vertices
            when public.ST_NPoints(geometry) &lt;= 2000
                then public.ST_AsText(public.ST_CollectionExtract(public.ST_MakeValid(geometry), 3))
            -- many vertices --&gt; use convex hull to simplify
            when public.ST_NPoints(geometry) &gt; 2000 and public.ST_NPoints(geometry) &lt;= 10000 then
                public.ST_AsText(public.ST_Multi(public.ST_ConvexHull(geometry)))
            -- way too many vertices --&gt; just use bounding box
            else public.ST_AsText(public.ST_Multi(public.ST_Envelope(geometry)))
          end as WKT
        from data_preparation.projects
        WHERE project_id = {project_id};
    """

    command = f"""
            export MAVEN_OPTS="-Xms4G -Xmx32G --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED --add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.sql/java.sql=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED"
            mvn compile exec:java\
            -Dexec.mainClass="{main_class}"\
            -Dexec.args="'{query}' {timestamp_start} {timestamp_end} {interval} {TEMP_DATA_PATH} {ignite_prefix} {project_id}"
            """

    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, cwd=cwd)
    # keep checking stdout/stderr until the child exits
    if oshdb_logs:
        while process.poll() is None:
            check_io(process)

    process.stdout.close()
    process.wait()
    logger.info("finished oshdb query for projects")
    logger.info(f"return code: {process.returncode} for project {project_id}")
</pre></body></html>