<p><div><div><div><div data-show-faces="false" data-height="21" data-width="100" data-layout="button_count" data-send="true" data-href="https://www.r-bloggers.com/r-interface-to-spark/"></div></div><div></div></div><div></div></div>
<div>
(This article was first published on <strong><a href="https://statcompute.wordpress.com/2017/06/09/r-interface-to-spark/"> S+/R – Yet Another Blog in Statistical Computing</a></strong>, and kindly contributed to <a rel="nofollow" href="https://www.r-bloggers.com/">R-bloggers)</a> </div>
<p><strong>SparkR</strong></p>
<pre>
library(SparkR, lib.loc = paste(Sys.getenv(&quot;SPARK_HOME&quot;), &quot;/R/lib&quot;, sep = &quot;&quot;))
sc &lt;- sparkR.session(master = &quot;local&quot;)
df1 &lt;- read.df(&quot;nycflights13.csv&quot;, source = &quot;csv&quot;, header = &quot;true&quot;, inferSchema = &quot;true&quot;)

### SUMMARY TABLE WITH SQL
createOrReplaceTempView(df1, &quot;tbl1&quot;)
summ &lt;- sql(&quot;select month, avg(dep_time) as avg_dep, avg(arr_time) as avg_arr from tbl1 where month in (1, 3, 5) group by month&quot;)
head(summ)
#   month  avg_dep  avg_arr
# 1     1 1347.210 1523.155
# 2     3 1359.500 1509.743
# 3     5 1351.168 1502.685

### SUMMARY TABLE WITH AGG()
grp &lt;- groupBy(filter(df1, &quot;month in (1, 3, 5)&quot;), &quot;month&quot;)
summ &lt;- agg(grp, avg_dep = avg(df1$dep_time), avg_arr = avg(df1$arr_time))
head(summ)
#   month  avg_dep  avg_arr
# 1     1 1347.210 1523.155
# 2     3 1359.500 1509.743
# 3     5 1351.168 1502.685
</pre>
<p><strong>sparklyr</strong></p>
<pre>
library(sparklyr)
sc &lt;- spark_connect(master = &quot;local&quot;)
df1 &lt;- spark_read_csv(sc, name = &quot;tbl1&quot;, path = &quot;nycflights13.csv&quot;, header = TRUE, infer_schema = TRUE)

### SUMMARY TABLE WITH SQL
library(DBI)
summ &lt;- dbGetQuery(sc, &quot;select month, avg(dep_time) as avg_dep, avg(arr_time) as avg_arr from tbl1 where month in (1, 3, 5) group by month&quot;)
head(summ)
#   month  avg_dep  avg_arr
# 1     5 1351.168 1502.685
# 2     1 1347.210 1523.155
# 3     3 1359.500 1509.743

### SUMMARY TABLE WITH DPLYR
library(dplyr)
summ &lt;- df1 %>% 
        filter(month %in% c(1, 3, 5)) %>% 
        group_by(month) %>%
        summarize(avg_dep = mean(dep_time), avg_arr = mean(arr_time)) 
head(summ)        
#   month  avg_dep  avg_arr
#   </pre>