title: Using airflow to uploade data on S3
body: <p>I tried to upload a dataframe containing informations about apple stock (using their api) as csv on s3 using airflow and pythonoperator. The script is below. When launched the dags appears as success but nothing happen at s3 level.
  I created 3 tasks one for gathering data another for creating s3 bucket and the last for uploading dataframe to S3 as csv file. The script works well in pure python.</p>
  <pre><code>import urllib
  from datetime import datetime
  from datetime import timedelta
  import pandas
  from urllib.request import urlopen
  from io import BytesIO
  
  
  import logging
  import boto3
  from botocore.exceptions import ClientError
  
  from airflow import DAG
  from airflow.operators.bash_operator import BashOperator
  from airflow.operators.python import PythonOperator, PythonVirtualenvOperator
  from airflow.utils.dates import days_ago
  from airflow.providers.amazon.aws.hooks.s3 import S3Hook
  s3_hook = S3Hook(aws_conn_id=&quot;my_conn_S3&quot;, region_name=&quot;eu-central-1&quot;)
  
  buket_name = &quot;BUCKET_NAME&quot;
  
  default_args = {
      'owner': 'admin',
      'depends_on_past': False,
      'start_date': datetime.now(),
      'email': name@email.com
      'email_on_failure': True,
      'email_on_retry': False,
      'retries': 2,
      'retry_delay': timedelta(minutes=0.5),
  }
  dag = DAG(
      dag_id='apple_finance',
      default_args=default_args,
      schedule_interval=None,
      start_date=days_ago(2),
      tags=['apple-finance'],
  )
  
  
  def get_apple_data():
      api_key = &quot;MY_API_KEY&quot;
      #get quotes
  
      quotes_url = f&quot;https://financialmodelingprep.com/api/v3/profile/AAPL?apikey={api_key}&quot;
      response = urlopen(quotes_url)
      quotes_data_ = response.read().decode(&quot;utf-8&quot;)
      quotes_data = json.loads(quotes_data_)
      pro_data= dict({key: val for key ,val in quotes_data[0].items() if key in ['companyName','price']})
  
      #gets_ratings
      rates_url = f&quot;https://financialmodelingprep.com/api/v3/rating/AAPL?apikey={api_key}&quot;
      response_r = urlopen(rates_url)
      rates_data_ = response_r.read().decode(&quot;utf-8&quot;)
      rates_data = json.loads(rates_data_)[0]
  
      rates_data = dict({key:val for key, val in rates_data.items() if key in ['rating','ratingScore', 'ratingRecommendation']}) 
      
      pro_data.update(rates_data)
  
      df = pandas.DataFrame(pro_data , index = [0])
      df['timestamp'] = datetime.now().isoformat()
      #df.to_csv(&quot;apple_data.csv&quot;)
      return df
  
  
  t1 = PythonOperator(
      task_id=&quot;get-APPL-data&quot;,
      python_callable = get_apple_data,
      dag=dag
  )
  
  def create_bucket():
      region = &quot;eu-central-1&quot;
      logger = logging.getLogger(&quot;airflow.task&quot;)
      try:
  
  
          s3_hook.create_bucket(bucket_name = bucket_name, region_name= region)
  
          #message = 
          logger.info(&quot;--------BUCKET  CREATED---------&quot;)
          
      except ClientError as e:
  
          logger.error(e)
          logger.info(&quot;----------CAN'T CREATE BUCKET------------&quot;)
          
          return False
      return True
  
  
  t2 = PythonOperator(
      task_id=&quot;create-s3-bucket&quot;,
      python_callable=create_bucket,
      dag=dag
  )
  
  def sendDataToS3(**kwargs):
  
      ti = kwargs['ti']
  
      apple_df = ti.xcom_pull(task_ids=&quot;get-APPL-data&quot;)
     
      region = &quot;eu-central-1&quot;
      #print(apple_df)
      csv_buffer = BytesIO
      apple_df.to_csv(csv_buffer)
  
      s3_hook._upload_file_obj(file_obj = csv_buffer.getvalue(), key = f&quot;apple_data_{datetime.now()}.csv&quot;,
      bucket_name = bucket_name)
      #s3_resource.Object(bucket, f&quot;apple_data_{datetime.now()}.csv&quot;).put(Body=csv_buffer.getvalue())
  
  
  t3 = PythonOperator(
      task_id=&quot;UploadToS3&quot;,
      python_callable=sendDataToS3,
      dag=dag
  )
  
  t0 &gt;&gt; t1 &gt;&gt; t2 &gt;&gt; t3 ```
  </code></pre>