Feat/sentiment analysis

macros/udfs/sentiment_analysis.sql

@@ -0,0 +1,36 @@
+{% macro create_udf_sentiment_analysis() %}
+
+create or replace function {{target.schema}}.udf_sentiment_analysis(text STRING, output INTEGER)
+returns STRING
+language PYTHON
+runtime_version = 3.8
+packages = ('pytorch','transformers','gensim')
+handler = 'sentiment_analysis_py'
+as
+
+$$
+
+from transformers import pipeline
+from gensim.parsing.preprocessing import remove_stopwords
+import string
+
+model = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+-- model = 'nlptown/bert-base-multilingual-uncased-sentiment'
+
+def sentiment_analysis(text, model_id, output='score'):
+
+    -- text pre-processing
+    text = str(remove_stopwords(text).translate(str.maketrans('', '', string.punctuation)))    
+
+    -- Classifier
+    classifier = pipeline("sentiment-analysis",model=model_id)
+    results = classifier(text)
+
+    if output=='label':
+        return results[0].get('label')
+    else:
+        return results[0].get('score')
+
+$$
+
+{% endmacro %}

macros/udfs/sentiment_analysis.yml

@@ -0,0 +1,31 @@
+version: 2
+
+macros:
+  - name: create_udf_sentiment_analysis
+    description: "
+      This macro iterates through a piece of text to return the overall 
+      sentiment of that text. 
+      First, the macro pre-processes the text removing unnecessary punctuation
+      and stopwords to help increase the accuracy of the model. Subsequently, using
+      the transformers library it applies a sentiment analysis pipeline based on a 
+      pre-trained model that will return either a score or a label for the text.
+
+      Recommendation is to use the following popular models:
+      1. cardiffnlp/twitter-roberta-base-sentiment-latest:
+          (https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment-latest)
+          This model is trained on 124M tweets from January 2018 to December 2021,
+          and is finetuned for sentiment analysis.
+          It outputs a label - Neutral, Positive or Negative - and a score ranging
+          from 0 to 1 - 0 being the most negative and 1, the most positive.
+      2. nlptown/bert-base-multilingual-uncased-sentiment:
+          (https://huggingface.co/nlptown/bert-base-multilingual-uncased-sentiment)
+          This model is fine-tuned for sentiment analysis on product reviews in six 
+          languages: English, Dutch, German, French, Spanish and Italian. It outputs
+          a label - 1 to 5 stars - and a score ranging from 0 to 1 - 0 being the
+          most negative and 1, the most positive.
+
+      Macro returns a STRING data type. If 'score' is used as an output, then it will
+      have to be cast to FLOAT data type.
+      "
+    docs:
+      show: false