open-metadata--openmetadata/ingestion/Dockerfile.ci

FROM mysql:8.3 AS mysql

FROM apache/airflow:3.2.1-python3.10
USER root
RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \
  && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list

# Install Dependencies (listed in alphabetical order)
RUN dpkg --configure -a \
  && apt-get -qq update \
  && apt-get -qq install -y \
  alien \
  build-essential \
  default-libmysqlclient-dev \
  freetds-bin \
  freetds-dev \
  gcc \
  gnupg \
  libaio1 \
  libevent-dev \
  libffi-dev \
  libpq-dev \
  librdkafka-dev \
  libsasl2-dev \
  libsasl2-2 \
  libsasl2-modules \
  libsasl2-modules-gssapi-mit \
  libssl-dev \
  libxml2 \
  libkrb5-dev \
  default-jdk \
  openssl \
  # To ensure compatibility with unixodbc package
  odbcinst=2.3.11-2+deb12u1 \
  postgresql \
  postgresql-contrib \
  tdsodbc \
  unixodbc=2.3.11-2+deb12u1 \
  unixodbc-dev=2.3.11-2+deb12u1 \
  unzip \
  vim \
  git \
  wget --no-install-recommends \
  # Accept MSSQL ODBC License
  && ACCEPT_EULA=Y apt-get -qq install -y msodbcsql18 \
  # Pull bookworm-security fixes for known-vulnerable packages
  # (gnutls28 -> 3.7.9-2+deb12u7, libcap2 -> 1:2.66-4+deb12u3,
  # openssh -> 1:9.2p1-2+deb12u10, rsync -> 3.2.7-1+deb12u5).
  # --only-upgrade is a no-op for packages not present in the base image.
  && apt-get -qq install -y --only-upgrade \
       libgnutls30 libcap2 libcap2-bin openssh-client rsync sudo \
       libpam0g libpam-modules libpam-modules-bin libpam-runtime \
  && apt-get -qq purge -y \
       'imagemagick*' 'libmagick*' 'graphicsmagick*' \
  && apt-get -qq autoremove -y --purge \
  && rm -rf /var/lib/apt/lists/*
COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump

RUN if [ $(uname -m) = "arm64" ] | [ $(uname -m) = "aarch64" ]; \
  then \
  wget https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip -O /oracle-instantclient.zip && \
  unzip -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \
  else \
  wget https://download.oracle.com/otn_software/linux/instantclient/1917000/instantclient-basic-linux.x64-19.17.0.0.0dbru.zip -O /oracle-instantclient.zip && \
  unzip -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \
  fi

ENV LD_LIBRARY_PATH=/instantclient

# Install DB2 iAccess Driver
# Mirrored on cdn.getcollate.io to decouple builds from IBM's CDN availability.
# Use dpkg --force-depends because the .deb declares old Debian package names
# (libodbc1, odbcinst1debian2) that don't exist in Debian 12; the actual
# libraries (unixodbc, odbcinst) are installed earlier. SHA256 pinned to v29.
RUN if [ $(uname -m) = "x86_64" ]; then \
  wget -q https://cdn.getcollate.io/deps/ingestion/ibm/ibm-iaccess-1.1.0.29-1.0.amd64.deb -O /tmp/ibm-iaccess.deb \
  && echo "e60e968d2cee96b2851964456f5b31ab990b1aa47d8f2399607809f7d4514f58  /tmp/ibm-iaccess.deb" | sha256sum -c - \
  && dpkg -i --force-depends /tmp/ibm-iaccess.deb \
  && apt-get install -f -y --no-install-recommends \
  && rm -f /tmp/ibm-iaccess.deb; \
  fi

# Required for Starting Ingestion Container in Docker Compose
# Provide Execute Permissions to shell script
COPY --chown=airflow:0 --chmod=775 ingestion/ingestion_dependency.sh /opt/airflow
# Required for Ingesting Sample Data
COPY --chown=airflow:0 ingestion /home/airflow/ingestion
COPY --chown=airflow:0 openmetadata-spec /home/airflow/openmetadata-spec
COPY --chown=airflow:0 scripts/datamodel_generation.py /home/airflow/scripts/datamodel_generation.py

COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airflow-apis
# Required for Airflow DAGs of Sample Data
COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags
COPY --chown=airflow:0 ingestion/examples/airflow/test_dags /opt/airflow/dags
COPY --chown=airflow:0 ingestion/airflow-constraints-3.2.1.txt /home/airflow/airflow-constraints-3.2.1.txt

USER airflow

# Disable pip cache dir
# https://pip.pypa.io/en/stable/topics/caching/#avoiding-caching
ENV PIP_NO_CACHE_DIR=1
# Make pip silent
ENV PIP_QUIET=1

# Pin setuptools<81 as pkg_resources was removed in setuptools 81.0.0+
# cx-Oracle's setup.py still uses pkg_resources
RUN pip install --upgrade pip "setuptools<81"

# Pre-install cx-Oracle without build isolation to use the pinned setuptools
RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9"

# Install FAB provider for Airflow 3.x Flask Blueprint compatibility
RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.2.1.txt" || true


WORKDIR /home/airflow/openmetadata-airflow-apis
RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.2.1.txt"

WORKDIR /home/airflow/ingestion

# Pre-install dialect packages that declare SQLAlchemy<2 in their metadata
# but work fine at runtime with SQLAlchemy 2.0 (unmaintained packages).
RUN pip install --no-deps "sqlalchemy-redshift==0.8.14" "sqlalchemy-ibmi==0.9.3" "pydoris-custom==1.1.0"
RUN pip install "datamodel-code-generator==0.25.6"
RUN mkdir -p /home/airflow/ingestion/src/metadata/generated
RUN python /home/airflow/scripts/datamodel_generation.py

# Argument to provide for Ingestion Dependencies to install. Defaults to all
ARG INGESTION_DEPENDENCY="all"
RUN pip install ".[${INGESTION_DEPENDENCY}]"

# Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593
RUN [ $(uname -m) = "x86_64" ] \
  && pip install ".[db2]" \
  || echo "DB2 not supported on ARM architectures."

# bump python-daemon for https://github.com/apache/airflow/pull/29916
RUN pip install "python-daemon>=3.0.0"

# remove all airflow providers except for docker, cncf kubernetes, and standard (required in Airflow 3.x)
RUN pip freeze | grep "apache-airflow-providers" | grep --invert-match -E "docker|http|cncf|fab|common|standard" | xargs pip uninstall -y

# Uninstalling psycopg2-binary and installing psycopg2 instead 
# because the psycopg2-binary generates a architecture specific error 
# while authenticating connection with the airflow, psycopg2 solves this error
RUN pip uninstall psycopg2-binary -y
RUN pip install psycopg2 mysqlclient==2.1.1
# Make required folders for openmetadata-airflow-apis
RUN mkdir -p /opt/airflow/dag_generated_configs

EXPOSE 8080
# Airflow 3.2.1 requires universal-pathlib>=0.3.8, but prior installs in this image
# can leave stale 0.2.6 `upath` module files in site-packages that cause import
# errors at runtime. Force-remove the stale registration then pin to the required version.
RUN pip uninstall upath -y && pip install "universal-pathlib==0.3.10"

# This is required as it's responsible to create airflow.cfg file
RUN airflow db migrate && rm -f /opt/airflow/airflow.db