Comment déployer un job ETL avancé avec Talend en 2026

#!/bin/bash

# Téléchargement de TOSDI 8.0.1 (adaptez l'URL à la dernière version)
wget https://github.com/Talend/open-studio-dist/releases/download/8.0.1-R2023-08/TOS_DI-8.0.1-20230817-1240-V8.0.1.zip -O tosdi.zip

# Décompression et lancement (Linux/Mac, adaptez pour Windows)
unzip tosdi.zip
cd TOS_DI-8.0.1-20230817-1240-V8.0.1

# Lancement avec 4GB heap pour jobs complexes
./Talend-Studio-linux64 &

# Vérification : Studio s'ouvre sur splash screen
# Créez un nouveau projet 'ETL_Advanced'

db_host=your-postgres-host
db_port=5432
db_user=postgres
db_password=secret123
db_name=clients_db
s3_bucket=your-s3-bucket
s3_key=access-key
s3_secret=secret-key
spark_master=yarn
spark_app_name=ProcessClients
log_level=INFO
batch_size=100000

package routines;

import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.codec.digest.DigestUtils;

public class Calculs {

    public static String hashClientId(String clientId) {
        if (clientId == null || clientId.isEmpty()) return "anonymous";
        return DigestUtils.sha256Hex(clientId + "salt2026");
    }

    public static Double computeChurnScore(Integer age, Double revenue) {
        if (age == null || revenue == null) return 0.5;
        Double score = (age > 60 ? 0.3 : 0.1) + (revenue < 50000 ? 0.4 : 0.0);
        return Math.min(1.0, score);
    }

    public static String categorizeRisk(Double score) {
        if (score > 0.7) return "HIGH";
        if (score > 0.4) return "MEDIUM";
        return "LOW";
    }
}

-- Créez la table test si absente
CREATE TABLE IF NOT EXISTS clients (
  client_id VARCHAR(50) PRIMARY KEY,
  age INT,
  revenue DECIMAL(10,2),
  region VARCHAR(20),
  updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

-- Insérez 1M lignes de test (utilisez generate_series pour prod)
INSERT INTO clients (client_id, age, revenue, region)
SELECT
  'client_' || generate_series(1,1000000),
  20 + (random() * 60)::INT,
  20000 + (random() * 100000)::DECIMAL(10,2),
  ARRAY['EU','US','ASIA'][floor(random()*3)::INT]
FROM generate_series(1,1000) g;

-- Query du job
SELECT client_id, age, revenue::DECIMAL(10,2), region, updated_at
FROM clients
WHERE updated_at > CURRENT_DATE - INTERVAL '7 days';

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.example</groupId>
  <artifactId>ProcessClients</artifactId>
  <version>1.0</version>
  <packaging>jar</packaging>
  <properties>
    <maven.compiler.source>11</maven.compiler.source>
    <maven.compiler.target>11</maven.compiler.target>
    <talend.job.version>1.0</talend.job.version>
  </properties>
  <dependencies>
    <dependency>
      <groupId>org.talend</groupId>
      <artifactId>talend-job</artifactId>
      <version>8.0.1</version>
      <scope>system</scope>
      <systemPath>${talend.studio.path}/poms/code/org.talend.designer.codegen-8.0.1.jar</systemPath>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-codec</artifactId>
      <version>1.15</version>
    </dependency>
    <!-- Ajoutez AWS SDK, Spark, Postgres JDBC -->
    <dependency>
      <groupId>org.postgresql</groupId>
      <artifactId>postgresql</artifactId>
      <version>42.7.1</version>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>s3</artifactId>
      <version>2.26.8</version>
    </dependency>
  </dependencies>
  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.5.1</version>
        <executions>
          <execution>
            <phase>package</phase>
            <goals><goal>shade</goal></goals>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
</project>

#!/bin/bash

# Après export Talend, dans dossier code/generated
mvn clean install -DskipTests -Pspark

# Vérification
ls -la target/ProcessClients-*.jar

# Taille ~150MB pour Spark deps
# Copie vers serveur prod
scp target/ProcessClients-1.0.jar user@prod-server:/opt/talend/jobs/

#!/bin/bash
export TALEND_CONTEXT=prod
export TALEND_PROPERTIES_PATH=/opt/talend/context_prod.properties

# Standalone avec heap 8GB, Spark local[*] ou yarn
java -Xmx8g \
  -Dfile.encoding=UTF-8 \
  -Dlog4j.configurationFile=log4j2.xml \
  -cp "ProcessClients-1.0.jar:lib/*" \
  com.example.ProcessClients \
  --context=prod \
  --context_param db_host=prod-db.example.com \
  --business_name=clients_$(date +%Y%m%d)

# Monitoring : tail -f logs/talend.log | grep ERROR