user709: I have a device built with an Arduino uno:

Arduino software installed on an Arduino uno
can be controlled via serial commands
can be controlled via physical buttons and sensors
on any button/sensor state change it writes the current state to serial
if no message has been sent within 5 seconds, it sends a no-change serial message


What is needed:

Use an ESP8266 to provide a bridge between current Arduino software and MQTT / web
I can program the ESP8266 as a web server, MQTT client, etc with the Arduino IDE or Lua (but I prefer the Arduino IDE since I can reuse parts of the code to generate / interpret communication). 
the ESP8266 will handle everything that is required for wifi / web / MQTT; without the MQTT module the Arduino part will work stand-alone, only the remote control will be missing.
I would like to make minimal changes to the Arduino code (or none, if possible). Any changes would involve extensive re-testing, which I try to avoid.
the ESP8266 can be missing in some installs.


What options I found:

Serial

ESP8266 can read the serial output and be a bridge between web/MQTT and serial, will store the current state in memory to be sent when requested to avoid polling the device every time.
One of the benefits is no code changes / testing required for the Arduino part.

I2C

Make the Arduino the I2C master and the ESP8266 the slave (or the other way around) and implement bi-directional communication. Got the idea by reading this thread.

Some other information about the serial commands:
A data packet (command or state description) consists of 1 to 20 characters, with a possible peak of 20 packets in 5 seconds, and an average of one packet every 3 seconds. If needed, I can make this send 5 unsigned integers instead of alphanumeric characters.
If more than the I2C / serial pins are needed I can upgrade to an Arduino Mega (so the number of free pins is not an issue).
Are there any other options for this? (protocols, pre-made libraries for serial communication, etc). I'm trying not to reinvent the wheel..
Thank you for your time!

user30341: Using I2C (which i recommend over serial), you may not use a NodeMCU ESP8266 as a slave.  It can only function as master.  An Arduino, on the other hand, may be either.  Therefore, the Arduino will always be passive and the NodeMCU will be the master.
What you can do, is implement a semaphore (as described by Majenko here), where the Arduino signals that it has something to "say" to the master, causing the master to request from the Arduino.  It could, under these circumstances, be labeled an "obliging master" (neologism).
NEVER print to serial from inside requestEvent or receiveEvent.

user-1: Most I2C tutorials made each Arduino a slave and master, but this approach is better, because each Arduino is either a master or a slave (not both) and no switching is needed. That makes things easier.
I2C is better than serial, because you can add more Arduinos in the same bus.
I have implemented I2C between two Arduinos and it's not more complicated that reading/writing to the serial port (which you have already done). And I bet you can generalize your serial port code to work with both serial and I2C comms.
This is my example (just a proof of concept). The slave Arduino controls a some pins, a temp. sensor and a watchdog under orders from the master Arduino. If slave doesn't receive a beat in time, it reset the master Arduino. 
Master code
#include <Wire.h>

#define CMD_SENSOR    1
#define CMD_PIN_ON    2
#define CMD_PIN_OFF   3
#define CMD_LUMEN     4
#define CMD_BEAT      5

const byte SLAVE_ADDRESS = 42;
const byte LED = 13;

char buffer[32];

void setup ()
{
  Serial.begin(9600);
  Serial.println("Master");

  Wire.begin ();

  pinMode (LED, OUTPUT);
  digitalWrite(LED, HIGH);
  delay(1000);
  digitalWrite(LED, LOW);

  Wire.beginTransmission (SLAVE_ADDRESS);
  Serial.println("Send LED on");
  Wire.write (CMD_PIN_ON);
  Wire.write (2);
  Wire.write (10);
  Wire.endTransmission();
  int x = Wire.requestFrom(SLAVE_ADDRESS, 1);
  Serial.print("status=");
  Serial.println(x);
}  // end of setup

void loop ()
{
  Serial.println(".");
  Wire.beginTransmission (SLAVE_ADDRESS);
  Wire.write (CMD_SENSOR);
  Wire.endTransmission();
  int x = Wire.requestFrom(SLAVE_ADDRESS, 1);
  Serial.print("Disponibles = ");
  Serial.println(x);
  int temp = (int) Wire.read();
  Serial.println(temp);

  Wire.beginTransmission (SLAVE_ADDRESS);
  Wire.write (CMD_LUMEN);
  Wire.endTransmission();
  Wire.requestFrom(SLAVE_ADDRESS, 2);
  int light = Wire.read() << 8 | Wire.read();  
  Serial.print("Light=");
  Serial.println(light);

  Wire.beginTransmission (SLAVE_ADDRESS);
  Wire.write (CMD_BEAT);
  Wire.endTransmission();
  Wire.requestFrom(SLAVE_ADDRESS, 1);

  delay (5000);
}  // end of loop

Slave code
/*
   Esclavo I2C

   Recibe los siguientes comandos
   <- 1° byte -> <- 2° byte -> <- 3° byte ->
   CMD_SENSOR
   CMD_PIN_ON    n° de pin     duracion en segundos
   CMD_PIN_OFF   n° de pin
   CMD_LUMEN
   CMD_BEAT

   Cada comando recibe una respuesta, ya sea el valor pedido
   o un status.
*/
#include <max6675.h>
#include <Wire.h>

typedef struct {
  int pin;
  unsigned long off;
} PIN_PGMA;

/*
   Lista de pines que se pueden activar via CMD_PIN_ON.
*/
#define PIN_LUMEN   A0
#define PIN_LED      2
#define PIN_RESET    3
PIN_PGMA pgma[] = {
  {PIN_LED, 0},
  {PIN_RESET, 0}
};
const int pgmaSize = sizeof(pgma) / sizeof(PIN_PGMA);

#define CMD_SENSOR    1
#define CMD_PIN_ON    2
#define CMD_PIN_OFF   3
#define CMD_LUMEN     4
#define CMD_BEAT      5

#define ST_OK         0
#define ST_BAD_PIN    1
#define ST_TIME_0     2
#define ST_BAD_LEN    3

#define MY_ADDRESS  42

//  Maximo tiempo de espera entre comandos CMD_BEAT. Pasado
//  ese tiempo, se activa el PIN_RESET.
//  En milisegundos.
#define BEAT_INTERVAL 10000
unsigned long lastBeat;
//  Largo del reset en milisegundos.
#define RESET_LENGTH  250
byte cmd = 0;
byte status = 0;

int thermoDO = 11;
int thermoCS = 12;
int thermoCLK = 13;

MAX6675 thermocouple(thermoCLK, thermoCS, thermoDO);

void setup ()
{
  Serial.begin(9600);

  pinMode(PIN_LUMEN, INPUT);
  analogRead(PIN_LUMEN);

  for (int i = 0; i < pgmaSize; i++) {
    pinMode(pgma[i].pin, OUTPUT);
    digitalWrite(pgma[i].pin, LOW);
  }
  lastBeat = millis();

  Wire.begin (MY_ADDRESS);
  Wire.onReceive (receiveCommand);
  Wire.onRequest (sendAnswer);
} 

void loop()
{
  unsigned long now = millis();

  //  Baja la linea de RESET si no ha recibido un beat ultimamente.
  unsigned long diff = now - lastBeat;
  if (diff > BEAT_INTERVAL) {
    resetPin();
  }

  //  Recorre la lista de pines y apaga aquellos cuyo tiempo termino.
  for (int i = 0; i < pgmaSize; i++) {
    if (pgma[i].off > 0 && pgma[i].off <= now) {
      Serial.print("off pin="); Serial.println(pgma[i].pin);
      pgma[i].off = 0;
      digitalWrite(pgma[i].pin, LOW);
    }
  }
}

// called by interrupt service routine when outgoing data is requested
void sendAnswer() {
  byte temp;
  int  lightReading;

  switch (cmd) {
    case CMD_SENSOR:
      temp = thermocouple.readCelsius();
      Wire.write(temp);
      break;
    case CMD_LUMEN:
      lightReading  = analogRead(PIN_LUMEN);
      Wire.write(lightReading >> 8);
      Wire.write(lightReading % 0xFF);
      break;
    case CMD_PIN_ON:
    case CMD_PIN_OFF:
    case CMD_BEAT:
      Wire.write(status);
      status = ST_OK;
      break;
  }
  cmd = 0;
}

// called by interrupt service routine when incoming data arrives
void receiveCommand (int howMany) {
  cmd = Wire.read ();
  status = ST_OK;
  switch (cmd) {
    case CMD_PIN_ON:
      cmdPinOn();;
      break;
    case CMD_PIN_OFF:
      cmdPinOff();
      break;
    case CMD_BEAT:
      lastBeat = millis();
      break;
  }
}  // end of receiveEvent

void cmdPinOff() {
  if (Wire.available() != 1) {
    status = ST_BAD_LEN;
  } else {
    int pin = Wire.read();
    int i = searchPin(pin);
    if (i < 0) {
      status = ST_BAD_PIN;
    }
    else {
      pgma[i].off = 0;
      digitalWrite(pin, LOW);
    }
  }
}

int searchPin(int pin) {
  int i = pgmaSize - 1;
  while (i >= 0 && pgma[i].pin != pin) {
    i--;
  }
  return i;
}

/*
 * Programa el encendido y duracion del RESET.
 */
void resetPin() {  
  if (digitalRead(PIN_RESET) == LOW) {
    unsigned long now = millis();

    int i = searchPin(PIN_RESET);

    pgma[i].off = now + RESET_LENGTH;
    lastBeat = now;
    digitalWrite(PIN_RESET, HIGH);
  }
}

void cmdPinOn() {
  if (Wire.available() != 2) {
    status = ST_BAD_LEN;
  } else {
    int pin = Wire.read();
    int len = Wire.read();

    int i = searchPin(pin);
    Serial.print("pin="); Serial.print(pin); Serial.print(",index="); Serial.println(i);
    if (i < 0) {
      status = ST_BAD_PIN;
      Serial.println("bad pin");
    } else {
      if (len == 0) {
        status = ST_TIME_0;
        Serial.println("ban len");
      }
      else {
        pgma[i].off = millis() + len * 1000;
        digitalWrite(pin, HIGH);
        Serial.println("ok");
      }
    }
  }
}


709: <p>I have a device built with an Arduino uno:</p>

<ul>
<li><p>Arduino software installed on an Arduino uno</p></li>
<li><p>can be controlled via serial commands</p></li>
<li><p>can be controlled via physical buttons and sensors</p></li>
<li><p>on any button/sensor state change it writes the current state to serial</p></li>
<li><p>if no message has been sent within 5 seconds, it sends a no-change serial message</p></li>
</ul>

<hr>

<p>What is needed:</p>

<ul>
<li><p>Use an ESP8266 to provide a bridge between current Arduino software and MQTT / web</p></li>
<li><p>I can program the ESP8266 as a web server, MQTT client, etc with the Arduino IDE or Lua (but I prefer the Arduino IDE since I can reuse parts of the code to generate / interpret communication). </p></li>
<li><p>the ESP8266 will handle everything that is required for wifi / web / MQTT; without the MQTT module the Arduino part will work stand-alone, only the remote control will be missing.</p></li>
<li><p>I would like to make minimal changes to the Arduino code (or none, if possible). Any changes would involve extensive re-testing, which I try to avoid.</p></li>
<li><p>the ESP8266 can be missing in some installs.</p></li>
</ul>

<hr>

<p>What options I found:</p>

<ol>
<li>Serial</li>
</ol>

<p>ESP8266 can read the serial output and be a bridge between web/MQTT and serial, will store the current state in memory to be sent when requested to avoid polling the device every time.</p>

<p>One of the benefits is no code changes / testing required for the Arduino part.</p>

<ol start="2">
<li>I2C</li>
</ol>

<p>Make the Arduino the I2C master and the ESP8266 the slave (or the other way around) and implement bi-directional communication. Got the idea by reading <a href="https://arduino.stackexchange.com/questions/3042/i2c-2-way-communication-between-arduino-uno-and-arduino-mega">this thread</a>.</p>

<hr>

<p>Some other information about the serial commands:</p>

<p>A data packet (command or state description) consists of 1 to 20 characters, with a possible peak of 20 packets in 5 seconds, and an average of one packet every 3 seconds. If needed, I can make this send 5 unsigned integers instead of alphanumeric characters.</p>

<p>If more than the I2C / serial pins are needed I can upgrade to an Arduino Mega (so the number of free pins is not an issue).</p>

<p>Are there any other options for this? (protocols, pre-made libraries for serial communication, etc). I'm trying not to reinvent the wheel..</p>

<p>Thank you for your time!</p>
 30341: <p>Using I2C (which i recommend over serial), you may not use a NodeMCU ESP8266 as a slave.  It can only function as master.  An Arduino, on the other hand, may be either.  Therefore, the Arduino will always be passive and the NodeMCU will be the master.</p>

<p>What you can do, is implement a semaphore (<a href="https://arduino.stackexchange.com/a/67462/30341">as described by Majenko here</a>), where the Arduino signals that it has something to "say" to the master, causing the master to request from the Arduino.  It could, under these circumstances, be labeled an "obliging master" (neologism).</p>

<p>NEVER print to serial from inside <code>requestEvent</code> or <code>receiveEvent</code>.</p>
 -1: <p>Most I2C tutorials made each Arduino a slave and master, but <a href="http://www.berryjam.eu/2014/07/advanced-arduino-i2c-communication/" rel="nofollow noreferrer">this</a> approach is better, because each Arduino is either a master or a slave (not both) and no switching is needed. That makes things easier.</p>

<p>I2C is better than serial, because you can add more Arduinos in the same bus.</p>

<p>I have implemented I2C between two Arduinos and it's not more complicated that reading/writing to the serial port (which you have already done). And I bet you can generalize your serial port code to work with both serial and I2C comms.</p>

<p>This is my example (just a proof of concept). The slave Arduino controls a some pins, a temp. sensor and a watchdog under orders from the master Arduino. If slave doesn't receive a beat in time, it reset the master Arduino. </p>

<p><strong>Master code</strong></p>

<pre><code>#include &lt;Wire.h&gt;

#define CMD_SENSOR    1
#define CMD_PIN_ON    2
#define CMD_PIN_OFF   3
#define CMD_LUMEN     4
#define CMD_BEAT      5

const byte SLAVE_ADDRESS = 42;
const byte LED = 13;

char buffer[32];

void setup ()
{
  Serial.begin(9600);
  Serial.println("Master");

  Wire.begin ();

  pinMode (LED, OUTPUT);
  digitalWrite(LED, HIGH);
  delay(1000);
  digitalWrite(LED, LOW);

  Wire.beginTransmission (SLAVE_ADDRESS);
  Serial.println("Send LED on");
  Wire.write (CMD_PIN_ON);
  Wire.write (2);
  Wire.write (10);
  Wire.endTransmission();
  int x = Wire.requestFrom(SLAVE_ADDRESS, 1);
  Serial.print("status=");
  Serial.println(x);
}  // end of setup

void loop ()
{
  Serial.println(".");
  Wire.beginTransmission (SLAVE_ADDRESS);
  Wire.write (CMD_SENSOR);
  Wire.endTransmission();
  int x = Wire.requestFrom(SLAVE_ADDRESS, 1);
  Serial.print("Disponibles = ");
  Serial.println(x);
  int temp = (int) Wire.read();
  Serial.println(temp);

  Wire.beginTransmission (SLAVE_ADDRESS);
  Wire.write (CMD_LUMEN);
  Wire.endTransmission();
  Wire.requestFrom(SLAVE_ADDRESS, 2);
  int light = Wire.read() &lt;&lt; 8 | Wire.read();  
  Serial.print("Light=");
  Serial.println(light);

  Wire.beginTransmission (SLAVE_ADDRESS);
  Wire.write (CMD_BEAT);
  Wire.endTransmission();
  Wire.requestFrom(SLAVE_ADDRESS, 1);

  delay (5000);
}  // end of loop
</code></pre>

<p><strong>Slave code</strong></p>

<pre><code>/*
   Esclavo I2C

   Recibe los siguientes comandos
   &lt;- 1° byte -&gt; &lt;- 2° byte -&gt; &lt;- 3° byte -&gt;
   CMD_SENSOR
   CMD_PIN_ON    n° de pin     duracion en segundos
   CMD_PIN_OFF   n° de pin
   CMD_LUMEN
   CMD_BEAT

   Cada comando recibe una respuesta, ya sea el valor pedido
   o un status.
*/
#include &lt;max6675.h&gt;
#include &lt;Wire.h&gt;

typedef struct {
  int pin;
  unsigned long off;
} PIN_PGMA;

/*
   Lista de pines que se pueden activar via CMD_PIN_ON.
*/
#define PIN_LUMEN   A0
#define PIN_LED      2
#define PIN_RESET    3
PIN_PGMA pgma[] = {
  {PIN_LED, 0},
  {PIN_RESET, 0}
};
const int pgmaSize = sizeof(pgma) / sizeof(PIN_PGMA);

#define CMD_SENSOR    1
#define CMD_PIN_ON    2
#define CMD_PIN_OFF   3
#define CMD_LUMEN     4
#define CMD_BEAT      5

#define ST_OK         0
#define ST_BAD_PIN    1
#define ST_TIME_0     2
#define ST_BAD_LEN    3

#define MY_ADDRESS  42

//  Maximo tiempo de espera entre comandos CMD_BEAT. Pasado
//  ese tiempo, se activa el PIN_RESET.
//  En milisegundos.
#define BEAT_INTERVAL 10000
unsigned long lastBeat;
//  Largo del reset en milisegundos.
#define RESET_LENGTH  250
byte cmd = 0;
byte status = 0;

int thermoDO = 11;
int thermoCS = 12;
int thermoCLK = 13;

MAX6675 thermocouple(thermoCLK, thermoCS, thermoDO);

void setup ()
{
  Serial.begin(9600);

  pinMode(PIN_LUMEN, INPUT);
  analogRead(PIN_LUMEN);

  for (int i = 0; i &lt; pgmaSize; i++) {
    pinMode(pgma[i].pin, OUTPUT);
    digitalWrite(pgma[i].pin, LOW);
  }
  lastBeat = millis();

  Wire.begin (MY_ADDRESS);
  Wire.onReceive (receiveCommand);
  Wire.onRequest (sendAnswer);
} 

void loop()
{
  unsigned long now = millis();

  //  Baja la linea de RESET si no ha recibido un beat ultimamente.
  unsigned long diff = now - lastBeat;
  if (diff &gt; BEAT_INTERVAL) {
    resetPin();
  }

  //  Recorre la lista de pines y apaga aquellos cuyo tiempo termino.
  for (int i = 0; i &lt; pgmaSize; i++) {
    if (pgma[i].off &gt; 0 &amp;&amp; pgma[i].off &lt;= now) {
      Serial.print("off pin="); Serial.println(pgma[i].pin);
      pgma[i].off = 0;
      digitalWrite(pgma[i].pin, LOW);
    }
  }
}

// called by interrupt service routine when outgoing data is requested
void sendAnswer() {
  byte temp;
  int  lightReading;

  switch (cmd) {
    case CMD_SENSOR:
      temp = thermocouple.readCelsius();
      Wire.write(temp);
      break;
    case CMD_LUMEN:
      lightReading  = analogRead(PIN_LUMEN);
      Wire.write(lightReading &gt;&gt; 8);
      Wire.write(lightReading % 0xFF);
      break;
    case CMD_PIN_ON:
    case CMD_PIN_OFF:
    case CMD_BEAT:
      Wire.write(status);
      status = ST_OK;
      break;
  }
  cmd = 0;
}

// called by interrupt service routine when incoming data arrives
void receiveCommand (int howMany) {
  cmd = Wire.read ();
  status = ST_OK;
  switch (cmd) {
    case CMD_PIN_ON:
      cmdPinOn();;
      break;
    case CMD_PIN_OFF:
      cmdPinOff();
      break;
    case CMD_BEAT:
      lastBeat = millis();
      break;
  }
}  // end of receiveEvent

void cmdPinOff() {
  if (Wire.available() != 1) {
    status = ST_BAD_LEN;
  } else {
    int pin = Wire.read();
    int i = searchPin(pin);
    if (i &lt; 0) {
      status = ST_BAD_PIN;
    }
    else {
      pgma[i].off = 0;
      digitalWrite(pin, LOW);
    }
  }
}

int searchPin(int pin) {
  int i = pgmaSize - 1;
  while (i &gt;= 0 &amp;&amp; pgma[i].pin != pin) {
    i--;
  }
  return i;
}

/*
 * Programa el encendido y duracion del RESET.
 */
void resetPin() {  
  if (digitalRead(PIN_RESET) == LOW) {
    unsigned long now = millis();

    int i = searchPin(PIN_RESET);

    pgma[i].off = now + RESET_LENGTH;
    lastBeat = now;
    digitalWrite(PIN_RESET, HIGH);
  }
}

void cmdPinOn() {
  if (Wire.available() != 2) {
    status = ST_BAD_LEN;
  } else {
    int pin = Wire.read();
    int len = Wire.read();

    int i = searchPin(pin);
    Serial.print("pin="); Serial.print(pin); Serial.print(",index="); Serial.println(i);
    if (i &lt; 0) {
      status = ST_BAD_PIN;
      Serial.println("bad pin");
    } else {
      if (len == 0) {
        status = ST_TIME_0;
        Serial.println("ban len");
      }
      else {
        pgma[i].off = millis() + len * 1000;
        digitalWrite(pin, HIGH);
        Serial.println("ok");
      }
    }
  }
}
</code></pre>
