title: R: tm package on German text
body: <p>I want to perform Sentiment classification on German dataset, I am using the following code, which works fine with english text, but raising error in case of German text.</p>
  
  <p>Here is my code for the following:</p>
  
  <pre><code>#loading required libraries
  library(tm)
  library(readxl)
  library(data.table)
  library(plyr)
  library(dplyr)
  library(zoo)
  library(ggplot2)
  library(ranger)
  library(e1071)
  
  df&lt;- data.table(read_excel("data/German2datasets.xlsx", skip = 1))
  
  # An abstract function to preprocess a text column
  preprocess &lt;- function(text_column)
  {
    # Use tm to get a doc matrix
    corpus &lt;- Corpus(VectorSource(text_column))
    # all lower case
    corpus &lt;- tm_map(corpus, content_transformer(tolower))
    # remove punctuation
    corpus &lt;- tm_map(corpus, content_transformer(removePunctuation))
    # remove numbers
    corpus &lt;- tm_map(corpus, content_transformer(removeNumbers))
    # remove stopwords
    corpus &lt;- tm_map(corpus, removeWords, stopwords("german"))
    # stem document
    corpus &lt;- tm_map(corpus, stemDocument)
    # strip white spaces (always at the end)
    corpus &lt;- tm_map(corpus, stripWhitespace)
    # return
    corpus    
  }
  
  # Get preprocess training and test data
  corpus &lt;- preprocess(df$TEXT)
  
  
  # Create a Document Term Matrix for train and test
  # Just including bi and tri-grams
  
  Sys.setenv(JAVA_HOME='D://Program Files/Java/jre1.8.0_112') # for 32-bit version
  library(rJava)
  library(RWeka)
  
  # Bi-Trigram tokenizer function (you can always get longer n-grams)
  bitrigramtokeniser &lt;- function(x, n) {
    RWeka:::NGramTokenizer(x, RWeka:::Weka_control(min = 2, max = 3))
  }
  
  
  "
  Remove remove words &lt;=2
  TdIdf weighting
  Infrequent (&lt; than 1% of documents) and very frequent (&gt; 80% of documents) terms not included
  "
  
  dtm &lt;- DocumentTermMatrix(corpus, control=list(wordLengths=c(2, Inf), 
                                                 tokenize = bitrigramtokeniser, 
                                                 weighting = function(x) weightTfIdf(x, normalize = FALSE),
                                                 bounds=list(global=c(floor(length(corpus)*0.01), floor(length(corpus)*.8)))))
  
  
  sent &lt;- df$Sentiment
  
  # Variable selection
  # ~~~~~~~~~~~~~~~~~~~~
  "
  For dimension reduction.
  The function calculates chi-square value for each phrase and keeps phrases with highest chi_square values
  Ideally you want to put variable selection as part of cross-validation.
  
  chisqTwo function takes:
  document term matrix (dtm), 
  vector of labels (labels), and 
  number of n-grams you want to keep (n_out)
  
  "
  chisqTwo &lt;- function(dtm, labels, n_out=2000){
    mat       &lt;- as.matrix(dtm)
    cat1      &lt;-  colSums(mat[labels==T,])        # total number of times phrase used in cat1 
    cat2      &lt;-  colSums(mat[labels==F,])        # total number of times phrase used in cat2 
    n_cat1        &lt;-  sum(mat[labels==T,]) - cat1     # total number of phrases in soft minus cat1
    n_cat2        &lt;-  sum(mat[labels==F,]) - cat2     # total number of phrases in hard minus cat2
  
    num       &lt;- (cat1*n_cat2 - cat2*n_cat1)^2
    den       &lt;- (cat1 + cat2)*(cat1 + n_cat1)*(cat2 + n_cat2)*(n_cat1 + n_cat2)
    chisq         &lt;- num/den
  
    chi_order &lt;- chisq[order(chisq)][1:n_out]   
    mat       &lt;- mat[, colnames(mat) %in% names(chi_order)]
  
  }
  
  
  n &lt;- nrow(dtm)
  shuffled &lt;- dtm[sample(n),]
  train_dtm &lt;- shuffled[1:round(0.7 * n),]
  test_dtm &lt;- shuffled[(round(0.7 * n) + 1):n,]
  
  
  "
  With high dimensional data, test matrix may not have all the phrases training matrix has.
  This function fixes that - so that test matrix has the same columns as training.
  testmat takes column names of training matrix (train_mat_cols), and 
  test matrix (test_mat)
  and outputs test_matrix with the same columns as training matrix
  "
  # Test matrix maker
  testmat &lt;- function(train_mat_cols, test_mat){  
    # train_mat_cols &lt;- colnames(train_mat); test_mat &lt;- as.matrix(test_dtm)
    test_mat  &lt;- test_mat[, colnames(test_mat) %in% train_mat_cols]
  
    miss_names    &lt;- train_mat_cols[!(train_mat_cols %in% colnames(test_mat))]
    if(length(miss_names)!=0){
      colClasses  &lt;- rep("numeric", length(miss_names))
      df          &lt;- read.table(text = '', colClasses = colClasses, col.names = miss_names)
      df[1:nrow(test_mat),] &lt;- 0
      test_mat    &lt;- cbind(test_mat, df)
    }
    as.matrix(test_mat)
  }
  
  # Train and test matrices
  train_mat &lt;- chisqTwo(train_dtm, train$Sentiment)
  test_mat  &lt;- testmat(colnames(train_mat), as.matrix(test_dtm))
  
  dim(train_mat)
  dim(test_mat)
  
  
  n &lt;- nrow(df)
  shuffled &lt;- df[sample(n),]
  train_data &lt;- shuffled[1:round(0.7 * n),]
  test_data &lt;- shuffled[(round(0.7 * n) + 1):n,]
  
  train_mat &lt;- as.data.frame(as.matrix(train_mat))
  colnames(train_mat) &lt;- make.names(colnames(train_mat))
  train_mat$Sentiment &lt;- train_data$Sentiment
  
  test_mat &lt;- as.data.frame(as.matrix(test_mat))
  colnames(test_mat) &lt;- make.names(colnames(test_mat))
  test_mat$Sentiment &lt;- test_data$Sentiment
  
  train_mat$Sentiment &lt;- as.factor(train_mat$Sentiment)
  test_mat$Sentiment &lt;- as.factor(test_mat$Sentiment)
  </code></pre>
  
  <p>Then, I will apply caret ML algos on the same for prediction of the Sentiment on the train and test data created.</p>
  
  <p>I am getting the following error at "preprocess" function.</p>
  
  <pre><code>&gt; corpus &lt;- preprocess(df$TEXT)
   Show Traceback
  
   Rerun with Debug
   Error in FUN(content(x), ...) : 
    invalid input 'Ich bin seit Jahren zufrieden mit der Basler VersicherubgðŸŒº' in 'utf8towcs' 
  </code></pre>
  
  <p>Data - <a href="https://drive.google.com/open?id=1T_LpL2G8upztihAC2SQeVs4YCPH-yfOs" rel="nofollow noreferrer">https://drive.google.com/open?id=1T_LpL2G8upztihAC2SQeVs4YCPH-yfOs</a></p>