Getting refresh_token using googleapis lib

So I have Google Oauth flow setup using googleapis lib, and I don't seem tog et a refresh_token after I use oauthClient.getToken(code, ..). It only returns the access_token and the id_token.

Here's my code:

function getOauth() {
  var oauthClient = new google.auth.OAuth2(
    config.googleOauth.id,
    config.googleOauth.secret,
    `${config.https ? 'https' : 'http'}://${config.baseUrl}/google`
  );
  return Bluebird.promisifyAll(oauthClient);
}

/**
 * Initialize session based of the auth code, which
 * gives us the accessToken and the payload.
 *
 * Returns a session with the user tokenized and
 * the accessToken for use to restore on page refresh.
 */
router.get('/initial', function * (req, res) {
  try { 
    let oauth = getOauth();
    let code = req.query.authorizationCode;
    let results = yield oauth.getTokenAsync(code);
    let tokens = results[0];
    let result = yield oauth.verifyIdTokenAsync(tokens.id_token, config.googleOauth.id);
    let payload = result.getPayload();
    let user = yield User.findOneByEmail(payload.email, req.communityConfig);
    let data = { user };

    if (user.role) {
      let role = yield Role.findOne(user.role, req.communityConfig);
      data.roles = [role];
    }

    let token = auth.createToken(data);

    res.json(extend({}, req.query, { token, accessToken: tokens.id_token }));
  } catch(error) {
    console.log(error.message);
    throw new StatusError(401, error);
  }
});

/**
 * Use the google provider's fetch method to restore the session
 * data using the accessToken.
 *
 * Returns a token created from the user and role, along with
 * all of the query props passed in.
 */
router.get('/restore', function * (req, res) {
  try {
    let oauth = getOauth();

    oauth.setCredentials({
      access_token: req.query.accessToken
    });

    let tokens = yield oauth.getAccessTokenAsync();
    let result = yield oauth.verifyIdTokenAsync(tokens[0], config.googleOauth.id);
    let payload = result.getPayload();
    let user = yield User.findOneByEmail(payload.email, req.communityConfig);
    let data = { user };

    if (user.role) {
      let role = yield Role.findOne(user.role, req.communityConfig);
      data.roles = [role];
    }

    let token = auth.createToken(data);

    res.json(extend({}, req.query, { token }));
  } catch(error) {
    console.log(error.message);
    throw new StatusError(401, error);
  }
});

So everything works, on initial and on refresh, but after the access_token expires, it no longer authenticates.

Token used too late, 1475070618.739 > 1475005777

I hear that I have to specify access_type to 'offline' but I don't even know where to set that in my case.