diff --git a/Kernel/Net/IPv4Socket.cpp b/Kernel/Net/IPv4Socket.cpp
index 3f2704ec29..83b3d794f7 100644
--- a/Kernel/Net/IPv4Socket.cpp
+++ b/Kernel/Net/IPv4Socket.cpp
@@ -248,6 +248,9 @@ KResultOr<size_t> IPv4Socket::sendto(OpenFileDescription&, const UserOrKernelBuf
 KResultOr<size_t> IPv4Socket::receive_byte_buffered(OpenFileDescription& description, UserOrKernelBuffer& buffer, size_t buffer_length, int flags, Userspace<sockaddr*>, Userspace<socklen_t*>)
 {
     MutexLocker locker(mutex());
+
+    VERIFY(m_receive_buffer);
+
     if (m_receive_buffer->is_empty()) {
         if (protocol_is_disconnected())
             return 0;
@@ -408,6 +411,8 @@ bool IPv4Socket::did_receive(const IPv4Address& source_address, u16 source_port,
     auto packet_size = packet.size();
 
     if (buffer_mode() == BufferMode::Bytes) {
+        VERIFY(m_receive_buffer);
+
         size_t space_in_receive_buffer = m_receive_buffer->space_for_writing();
         if (packet_size > space_in_receive_buffer) {
             dbgln("IPv4Socket({}): did_receive refusing packet since buffer is full.", this);
@@ -764,4 +769,9 @@ void IPv4Socket::set_can_read(bool value)
         evaluate_block_conditions();
 }
 
+void IPv4Socket::drop_receive_buffer()
+{
+    m_receive_buffer = nullptr;
+}
+
 }
diff --git a/Kernel/Net/IPv4Socket.h b/Kernel/Net/IPv4Socket.h
index 786d813c99..97b8e65e23 100644
--- a/Kernel/Net/IPv4Socket.h
+++ b/Kernel/Net/IPv4Socket.h
@@ -91,6 +91,7 @@ protected:
     void set_peer_address(IPv4Address address) { m_peer_address = address; }
 
     static KResultOr<NonnullOwnPtr<DoubleBuffer>> try_create_receive_buffer();
+    void drop_receive_buffer();
 
 private:
     virtual bool is_ipv4() const override { return true; }
@@ -115,7 +116,7 @@ private:
 
     SinglyLinkedListWithCount<ReceivedPacket> m_receive_queue;
 
-    NonnullOwnPtr<DoubleBuffer> m_receive_buffer;
+    OwnPtr<DoubleBuffer> m_receive_buffer;
 
     u16 m_local_port { 0 };
     u16 m_peer_port { 0 };
diff --git a/Kernel/Net/TCPSocket.cpp b/Kernel/Net/TCPSocket.cpp
index 854deede4b..555b606045 100644
--- a/Kernel/Net/TCPSocket.cpp
+++ b/Kernel/Net/TCPSocket.cpp
@@ -43,6 +43,13 @@ void TCPSocket::set_state(State new_state)
         [[maybe_unused]] auto rc = set_so_error(KSuccess);
     }
 
+    if (new_state == State::TimeWait) {
+        // Once we hit TimeWait, we are only holding the socket in case there
+        // are packets on the way which we wouldn't want a new socket to get hit
+        // with, so there's no point in keeping the receive buffer around.
+        drop_receive_buffer();
+    }
+
     if (new_state == State::Closed) {
         closing_sockets().with_exclusive([&](auto& table) {
             table.remove(tuple());