diff --git a/ikvm_server.cpp b/ikvm_server.cpp
index 679f47c..98d45ef 100644
--- a/ikvm_server.cpp
+++ b/ikvm_server.cpp
@@ -1,15 +1,209 @@
 #include "ikvm_server.hpp"
 
+#include <rfb/rfbproto.h>
+
+#include <phosphor-logging/elog-errors.hpp>
+#include <phosphor-logging/elog.hpp>
+#include <phosphor-logging/log.hpp>
+#include <xyz/openbmc_project/Common/error.hpp>
+
 namespace ikvm
 {
 
+using namespace phosphor::logging;
+using namespace sdbusplus::xyz::openbmc_project::Common::Error;
+
 Server::Server(const Args& args, Input& i, Video& v) :
-    input(i), video(v)
+    pendingResize(false), frameCounter(0), numClients(0), input(i), video(v)
 {
+    std::string ip("localhost");
+    const Args::CommandLine& commandLine = args.getCommandLine();
+    int argc = commandLine.argc;
+
+    server = rfbGetScreen(&argc, commandLine.argv, video.getWidth(),
+                          video.getHeight(), Video::bitsPerSample,
+                          Video::samplesPerPixel, Video::bytesPerPixel);
+
+    if (!server)
+    {
+        log<level::ERR>("Failed to get VNC screen due to invalid arguments");
+        elog<InvalidArgument>(
+            xyz::openbmc_project::Common::InvalidArgument::ARGUMENT_NAME(""),
+            xyz::openbmc_project::Common::InvalidArgument::ARGUMENT_VALUE(""));
+    }
+
+    framebuffer.resize(
+        video.getHeight() * video.getWidth() * Video::bytesPerPixel, 0);
+
+    server->screenData = this;
+    server->desktopName = "OpenBMC IKVM";
+    server->frameBuffer = framebuffer.data();
+    server->newClientHook = newClient;
+
+    rfbStringToAddr(&ip[0], &server->listenInterface);
+
+    rfbInitServer(server);
+
+    rfbMarkRectAsModified(server, 0, 0, video.getWidth(), video.getHeight());
+
+    server->kbdAddEvent = Input::keyEvent;
+    server->ptrAddEvent = Input::pointerEvent;
+
+    processTime = (1000000 / video.getFrameRate()) - 100;
 }
 
 Server::~Server()
 {
+    rfbScreenCleanup(server);
+}
+
+void Server::resize()
+{
+    if (frameCounter > video.getFrameRate())
+    {
+        doResize();
+    }
+    else
+    {
+        pendingResize = true;
+    }
+}
+
+void Server::run()
+{
+    rfbProcessEvents(server, processTime);
+
+    if (server->clientHead)
+    {
+        input.sendReport();
+
+        frameCounter++;
+        if (pendingResize && frameCounter > video.getFrameRate())
+        {
+            doResize();
+            pendingResize = false;
+        }
+    }
+}
+
+void Server::sendFrame()
+{
+    char* data = video.getData();
+    rfbClientIteratorPtr it;
+    rfbClientPtr cl;
+
+    if (!data || pendingResize)
+    {
+        return;
+    }
+
+    it = rfbGetClientIterator(server);
+
+    while ((cl = rfbClientIteratorNext(it)))
+    {
+        ClientData* cd = (ClientData*)cl->clientData;
+        rfbFramebufferUpdateMsg* fu = (rfbFramebufferUpdateMsg*)cl->updateBuf;
+
+        if (!cd)
+        {
+            continue;
+        }
+
+        if (cd->skipFrame)
+        {
+            cd->skipFrame--;
+            continue;
+        }
+
+        if (cl->enableLastRectEncoding)
+        {
+            fu->nRects = 0xFFFF;
+        }
+        else
+        {
+            fu->nRects = Swap16IfLE(1);
+        }
+
+        fu->type = rfbFramebufferUpdate;
+        cl->ublen = sz_rfbFramebufferUpdateMsg;
+        rfbSendUpdateBuf(cl);
+
+        cl->tightEncoding = rfbEncodingTight;
+        rfbSendTightHeader(cl, 0, 0, video.getWidth(), video.getHeight());
+
+        cl->updateBuf[cl->ublen++] = (char)(rfbTightJpeg << 4);
+        rfbSendCompressedDataTight(cl, data, video.getFrameSize());
+
+        if (cl->enableLastRectEncoding)
+        {
+            rfbSendLastRectMarker(cl);
+        }
+
+        rfbSendUpdateBuf(cl);
+    }
+
+    rfbReleaseClientIterator(it);
+}
+
+void Server::clientGone(rfbClientPtr cl)
+{
+    Server* server = (Server*)cl->screen->screenData;
+
+    delete (ClientData*)cl->clientData;
+
+    if (server->numClients-- == 1)
+    {
+        rfbMarkRectAsModified(server->server, 0, 0, server->video.getWidth(),
+                              server->video.getHeight());
+    }
+}
+
+enum rfbNewClientAction Server::newClient(rfbClientPtr cl)
+{
+    Server* server = (Server*)cl->screen->screenData;
+
+    cl->clientData =
+        new ClientData(server->video.getFrameRate(), &server->input);
+    cl->clientGoneHook = clientGone;
+    if (!server->numClients++)
+    {
+        server->pendingResize = false;
+        server->frameCounter = 0;
+        server->video.start();
+    }
+
+    return RFB_CLIENT_ACCEPT;
+}
+
+void Server::doResize()
+{
+    rfbClientIteratorPtr it;
+    rfbClientPtr cl;
+
+    framebuffer.resize(
+        video.getHeight() * video.getWidth() * Video::bytesPerPixel, 0);
+
+    rfbNewFramebuffer(server, framebuffer.data(), video.getWidth(),
+                      video.getHeight(), Video::bitsPerSample,
+                      Video::samplesPerPixel, Video::bytesPerPixel);
+    rfbMarkRectAsModified(server, 0, 0, video.getWidth(), video.getHeight());
+
+    it = rfbGetClientIterator(server);
+
+    while ((cl = rfbClientIteratorNext(it)))
+    {
+        ClientData* cd = (ClientData*)cl->clientData;
+
+        if (!cd)
+        {
+            continue;
+        }
+
+        // delay video updates to give the client time to resize
+        cd->skipFrame = video.getFrameRate();
+    }
+
+    rfbReleaseClientIterator(it);
 }
 
 } // namespace ikvm
diff --git a/ikvm_server.hpp b/ikvm_server.hpp
index 67602b0..ff51cfc 100644
--- a/ikvm_server.hpp
+++ b/ikvm_server.hpp
@@ -4,6 +4,10 @@
 #include "ikvm_input.hpp"
 #include "ikvm_video.hpp"
 
+#include <rfb/rfb.h>
+
+#include <vector>
+
 namespace ikvm
 {
 
@@ -53,6 +57,22 @@
     Server(Server&&) = default;
     Server& operator=(Server&&) = default;
 
+    /* @brief Resizes the RFB framebuffer */
+    void resize();
+    /* @brief Executes any pending RFB updates and client input */
+    void run();
+    /* @brief Sends pending video frame to clients */
+    void sendFrame();
+
+    /*
+     * @brief Indicates whether or not video data is desired
+     *
+     * @return Boolean to indicate whether any clients need a video frame
+     */
+    inline bool wantsFrame() const
+    {
+        return server->clientHead;
+    }
     /*
      * @brief Get the Video object
      *
@@ -64,10 +84,38 @@
     }
 
   private:
+    /*
+     * @brief Handler for a client disconnecting
+     *
+     * @param[in] cl - Handle to the client object
+     */
+    static void clientGone(rfbClientPtr cl);
+    /*
+     * @brief Handler for client connecting
+     *
+     * @param[in] cl - Handle to the client object
+     */
+    static enum rfbNewClientAction newClient(rfbClientPtr cl);
+
+    /* @brief Performs the resize operation on the framebuffer */
+    void doResize();
+
+    /* @brief Boolean to indicate if a resize operation is on-going */
+    bool pendingResize;
+    /* @brief Number of frames handled since a client connected */
+    int frameCounter;
+    /* @brief Number of connected clients */
+    int numClients;
+    /* @brief Microseconds to process RFB events every frame */
+    long int processTime;
+    /* @brief Handle to the RFB server object */
+    rfbScreenInfoPtr server;
     /* @brief Reference to the Input object */
     Input& input;
     /* @brief Reference to the Video object */
     Video& video;
+    /* @brief Default framebuffer storage */
+    std::vector<char> framebuffer;
 };
 
 } // namespace ikvm