Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a9e36708e8 | |||
| c00df926a0 | |||
| 632d79dd57 | |||
| b296d73367 | |||
| 2200657ba7 | |||
| 07f7893a26 | |||
| 7f21b733ed | |||
| 9816426780 | |||
| 4fc0c438c8 | |||
| 49a30d3b44 | |||
| b71e8121e2 | |||
| 9c6db30550 | |||
| bc2017e67d | |||
| cd4a239f14 | |||
| 0d0de32d21 | |||
| 88ae539f54 | |||
| 78a5b21531 | |||
| e011931436 | |||
| 70d9a37a4e | |||
| 2ab2e07b9b | |||
| 9a9c65c24c | |||
| 8a11ce0103 | |||
| cf63683c9c | |||
| 92bd56aac0 |
15
go.mod
15
go.mod
@@ -4,4 +4,17 @@ go 1.24.5
|
|||||||
|
|
||||||
require github.com/gorilla/websocket v1.5.3
|
require github.com/gorilla/websocket v1.5.3
|
||||||
|
|
||||||
require git.neurocipta.com/rogerferdinan/custom-rwmutex v1.0.0 // indirect
|
require (
|
||||||
|
git.neurocipta.com/rogerferdinan/custom-rwmutex v1.0.0
|
||||||
|
github.com/google/uuid v1.6.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.neurocipta.com/rogerferdinan/safe-map v0.0.0-20251011004629-ab0b119a7c48
|
||||||
|
github.com/gobwas/ws v1.4.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/gobwas/httphead v0.1.0 // indirect
|
||||||
|
github.com/gobwas/pool v0.2.1 // indirect
|
||||||
|
)
|
||||||
|
|||||||
11
go.sum
11
go.sum
@@ -1,4 +1,15 @@
|
|||||||
git.neurocipta.com/rogerferdinan/custom-rwmutex v1.0.0 h1:KnNc40SrYsg0cksIIcQy/ca6bunkGADQOs1u7O/E+iY=
|
git.neurocipta.com/rogerferdinan/custom-rwmutex v1.0.0 h1:KnNc40SrYsg0cksIIcQy/ca6bunkGADQOs1u7O/E+iY=
|
||||||
git.neurocipta.com/rogerferdinan/custom-rwmutex v1.0.0/go.mod h1:9DvvHc2UZhBwEs63NgO4IhiuHnBNtTuBkTJgiMnnCss=
|
git.neurocipta.com/rogerferdinan/custom-rwmutex v1.0.0/go.mod h1:9DvvHc2UZhBwEs63NgO4IhiuHnBNtTuBkTJgiMnnCss=
|
||||||
|
git.neurocipta.com/rogerferdinan/safe-map v0.0.0-20251011004629-ab0b119a7c48 h1:4wXSbEuwFd2gycaaGP35bjUkKEEO6WcVfJ6cetEyT5s=
|
||||||
|
git.neurocipta.com/rogerferdinan/safe-map v0.0.0-20251011004629-ab0b119a7c48/go.mod h1:QtIxG0BYCCq8a5qyklpSHA8qWUvKr+mfl42qF9QxTc0=
|
||||||
|
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||||
|
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||||
|
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||||
|
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||||
|
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||||
|
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
|||||||
56
internal/error_response.go
Normal file
56
internal/error_response.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package internal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type StatusMessageBuilder struct {
|
||||||
|
statusCode *int
|
||||||
|
message *string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStatusMessage() *StatusMessageBuilder {
|
||||||
|
return &StatusMessageBuilder{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sm *StatusMessageBuilder) StatusCode(statusCode int) *StatusMessageBuilder {
|
||||||
|
sm.statusCode = &statusCode
|
||||||
|
return sm
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sm *StatusMessageBuilder) Message(message string) *StatusMessageBuilder {
|
||||||
|
sm.message = &message
|
||||||
|
return sm
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sm *StatusMessageBuilder) Build() *StatusMessage {
|
||||||
|
return &StatusMessage{
|
||||||
|
StatusCode: sm.statusCode,
|
||||||
|
Message: sm.message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatusMessage struct {
|
||||||
|
StatusCode *int
|
||||||
|
Message *string `json:"msg"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func ErrorResponse(w http.ResponseWriter, statusMessage *StatusMessage) error {
|
||||||
|
if statusMessage.StatusCode == nil {
|
||||||
|
return fmt.Errorf("status code is missing")
|
||||||
|
}
|
||||||
|
if statusMessage.Message == nil {
|
||||||
|
return fmt.Errorf("message is missing")
|
||||||
|
}
|
||||||
|
w.Header().Add("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(*statusMessage.StatusCode)
|
||||||
|
|
||||||
|
b, err := json.Marshal(statusMessage)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal JSON")
|
||||||
|
}
|
||||||
|
w.Write(b)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
16
internal/format.go
Normal file
16
internal/format.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
package internal
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
func FormatBytes(b uint64) string {
|
||||||
|
const unit = 1024
|
||||||
|
if b < unit {
|
||||||
|
return fmt.Sprintf("%d B", b)
|
||||||
|
}
|
||||||
|
div, exp := uint64(unit), 0
|
||||||
|
for n := b / unit; n >= unit; n /= unit {
|
||||||
|
div *= unit
|
||||||
|
exp++
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.2f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||||
|
}
|
||||||
144
internal/hub.go
144
internal/hub.go
@@ -1,63 +1,127 @@
|
|||||||
package internal
|
package internal
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/gorilla/websocket"
|
"github.com/gorilla/websocket"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
writeWait = 10 * time.Second
|
writeWait = 10 * time.Second
|
||||||
pongWait = 60 * time.Second
|
pongWait = 60 * time.Second
|
||||||
pingPeriod = 25 * time.Second
|
pingPeriod = (pongWait * 9) / 10
|
||||||
|
maxMessageSize = 512
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
minHighWaterMarkMapRebuild = 100
|
||||||
|
mapRebuildThreshold = 4
|
||||||
)
|
)
|
||||||
|
|
||||||
type Client struct {
|
type Client struct {
|
||||||
|
ID string
|
||||||
Conn *websocket.Conn
|
Conn *websocket.Conn
|
||||||
Send chan []byte
|
Send chan []byte
|
||||||
SubscribedPath string
|
SubscribedPath string
|
||||||
done chan struct{}
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewClient(conn *websocket.Conn, subscribedPath string) *Client {
|
func NewClient(conn *websocket.Conn, subscribedPath string) *Client {
|
||||||
return &Client{
|
return &Client{
|
||||||
|
ID: uuid.NewString(),
|
||||||
Conn: conn,
|
Conn: conn,
|
||||||
Send: make(chan []byte, 64),
|
Send: make(chan []byte, 256),
|
||||||
SubscribedPath: subscribedPath,
|
SubscribedPath: subscribedPath,
|
||||||
done: make(chan struct{}),
|
mu: sync.Mutex{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Hub struct {
|
type Hub struct {
|
||||||
|
path string
|
||||||
|
maxClients int
|
||||||
Clients map[*Client]bool
|
Clients map[*Client]bool
|
||||||
Broadcast chan []byte
|
Broadcast chan []byte
|
||||||
Register chan *Client
|
Register chan *Client
|
||||||
Unregister chan *Client
|
Unregister chan *Client
|
||||||
|
monitor *MemoryMonitor
|
||||||
|
highWaterMark int
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHub() *Hub {
|
func NewHub(path string, maxClients int) *Hub {
|
||||||
|
log.Printf("[%s] Hub created with max clients: %d", path, maxClients)
|
||||||
return &Hub{
|
return &Hub{
|
||||||
Broadcast: make(chan []byte),
|
path: path,
|
||||||
Register: make(chan *Client),
|
maxClients: maxClients,
|
||||||
Unregister: make(chan *Client),
|
monitor: NewMemoryMonitor(),
|
||||||
|
Broadcast: make(chan []byte, 256),
|
||||||
|
Register: make(chan *Client, maxClients),
|
||||||
|
Unregister: make(chan *Client, maxClients),
|
||||||
Clients: make(map[*Client]bool),
|
Clients: make(map[*Client]bool),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Hub) Run() {
|
// Run starts the hub event loop. It exits when ctx is cancelled.
|
||||||
|
func (h *Hub) Run(ctx context.Context) {
|
||||||
|
monitorTicker := time.NewTicker(MonitorInterval)
|
||||||
go func() {
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
monitorTicker.Stop()
|
||||||
|
// On shutdown, close every client's Send channel so WritePump sends
|
||||||
|
// a WebSocket close frame and exits. Also expire the read deadline so
|
||||||
|
// blocked ReadMessage() calls in ReadPump return immediately instead
|
||||||
|
// of waiting up to pongWait (60 s).
|
||||||
|
for client := range h.Clients {
|
||||||
|
close(client.Send)
|
||||||
|
client.Conn.SetReadDeadline(time.Now())
|
||||||
|
}
|
||||||
|
log.Printf("[%s] Hub stopped\n", h.path)
|
||||||
|
}()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.Printf("[%s] Hub shutting down\n", h.path)
|
||||||
|
return
|
||||||
|
|
||||||
case client := <-h.Register:
|
case client := <-h.Register:
|
||||||
h.Clients[client] = true
|
if len(h.Clients) >= h.maxClients {
|
||||||
log.Println("Client registered")
|
close(client.Send)
|
||||||
case c := <-h.Unregister:
|
client.Conn.Close()
|
||||||
if _, ok := h.Clients[c]; ok {
|
log.Printf("[%s] Rejected client %s (max %d reached)\n", h.path, client.ID, h.maxClients)
|
||||||
delete(h.Clients, c)
|
break
|
||||||
close(c.Send)
|
|
||||||
}
|
}
|
||||||
log.Println("Client Unregistered")
|
h.Clients[client] = true
|
||||||
|
if len(h.Clients) > h.highWaterMark {
|
||||||
|
h.highWaterMark = len(h.Clients)
|
||||||
|
}
|
||||||
|
log.Printf("[%s] Client registered %s\n", h.path, client.ID)
|
||||||
|
|
||||||
|
case client := <-h.Unregister:
|
||||||
|
if _, ok := h.Clients[client]; ok {
|
||||||
|
delete(h.Clients, client)
|
||||||
|
close(client.Send)
|
||||||
|
|
||||||
|
// Rebuild the map when the live set has dropped to less than
|
||||||
|
// 1/mapRebuildThreshold of the peak, so the old backing buckets
|
||||||
|
// are released to the GC.
|
||||||
|
if h.highWaterMark >= minHighWaterMarkMapRebuild &&
|
||||||
|
len(h.Clients) < h.highWaterMark/mapRebuildThreshold {
|
||||||
|
rebuilt := make(map[*Client]bool, len(h.Clients))
|
||||||
|
for c := range h.Clients {
|
||||||
|
rebuilt[c] = true
|
||||||
|
}
|
||||||
|
h.Clients = rebuilt
|
||||||
|
h.highWaterMark = len(h.Clients)
|
||||||
|
log.Printf("[%s] Clients map rebuilt: %d active clients\n", h.path, len(h.Clients))
|
||||||
|
}
|
||||||
|
log.Printf("[%s] Client Unregistered %s\n", h.path, client.ID)
|
||||||
|
}
|
||||||
|
|
||||||
case message := <-h.Broadcast:
|
case message := <-h.Broadcast:
|
||||||
for client := range h.Clients {
|
for client := range h.Clients {
|
||||||
select {
|
select {
|
||||||
@@ -65,8 +129,18 @@ func (h *Hub) Run() {
|
|||||||
default:
|
default:
|
||||||
close(client.Send)
|
close(client.Send)
|
||||||
delete(h.Clients, client)
|
delete(h.Clients, client)
|
||||||
|
log.Printf("[%s] Client %s removed (slow/disconnected)\n", h.path, client.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case <-monitorTicker.C:
|
||||||
|
current, peak := h.monitor.Snapshot()
|
||||||
|
clientLength := len(h.Clients)
|
||||||
|
if clientLength > 0 {
|
||||||
|
log.Printf("[%s] connected clients: %d | heap alloc: %s | peak heap alloc: %s",
|
||||||
|
h.path, clientLength, FormatBytes(current), FormatBytes(peak),
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -75,7 +149,10 @@ func (h *Hub) Run() {
|
|||||||
func WritePump(c *Client, h *Hub) {
|
func WritePump(c *Client, h *Hub) {
|
||||||
pingTicker := time.NewTicker(pingPeriod)
|
pingTicker := time.NewTicker(pingPeriod)
|
||||||
defer func() {
|
defer func() {
|
||||||
h.Unregister <- c
|
select {
|
||||||
|
case h.Unregister <- c:
|
||||||
|
default:
|
||||||
|
}
|
||||||
pingTicker.Stop()
|
pingTicker.Stop()
|
||||||
c.Conn.Close()
|
c.Conn.Close()
|
||||||
}()
|
}()
|
||||||
@@ -84,17 +161,32 @@ func WritePump(c *Client, h *Hub) {
|
|||||||
select {
|
select {
|
||||||
case message, ok := <-c.Send:
|
case message, ok := <-c.Send:
|
||||||
c.Conn.SetWriteDeadline(time.Now().Add(writeWait))
|
c.Conn.SetWriteDeadline(time.Now().Add(writeWait))
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
c.Conn.WriteMessage(websocket.CloseMessage, []byte{})
|
c.Conn.WriteMessage(websocket.CloseMessage, []byte{})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.Conn.WriteMessage(websocket.TextMessage, message); err != nil {
|
w, err := c.Conn.NextWriter(websocket.TextMessage)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Write(message)
|
||||||
|
|
||||||
|
// Flush any messages that queued up while we were writing.
|
||||||
|
n := len(c.Send)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
w.Write(<-c.Send)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
case <-pingTicker.C:
|
case <-pingTicker.C:
|
||||||
c.Conn.SetWriteDeadline(time.Now().Add(writeWait))
|
c.Conn.SetWriteDeadline(time.Now().Add(writeWait))
|
||||||
if err := c.Conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
|
|
||||||
|
if err := c.Conn.WriteMessage(websocket.PingMessage, nil); err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -103,28 +195,30 @@ func WritePump(c *Client, h *Hub) {
|
|||||||
|
|
||||||
func ReadPump(c *Client, h *Hub) {
|
func ReadPump(c *Client, h *Hub) {
|
||||||
defer func() {
|
defer func() {
|
||||||
h.Unregister <- c
|
select {
|
||||||
|
case h.Unregister <- c:
|
||||||
|
default:
|
||||||
|
}
|
||||||
c.Conn.Close()
|
c.Conn.Close()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
c.Conn.SetReadLimit(512)
|
c.Conn.SetReadLimit(maxMessageSize)
|
||||||
c.Conn.SetReadDeadline(time.Now().Add(pongWait))
|
c.Conn.SetReadDeadline(time.Now().Add(pongWait))
|
||||||
|
|
||||||
c.Conn.SetPongHandler(func(string) error {
|
c.Conn.SetPongHandler(func(string) error {
|
||||||
c.Conn.SetReadDeadline(time.Now().Add(pongWait))
|
c.Conn.SetReadDeadline(time.Now().Add(pongWait))
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
for {
|
for {
|
||||||
messageType, message, err := c.Conn.ReadMessage()
|
_, message, err := c.Conn.ReadMessage()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
|
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
|
||||||
log.Printf("WebSocket error: %v", err)
|
log.Printf("WebSocket error: %v", err)
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if messageType == websocket.TextMessage {
|
|
||||||
log.Printf("Received: %s\n", message)
|
log.Printf("Received: %s\n", message)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
60
internal/memory_monitor.go
Normal file
60
internal/memory_monitor.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package internal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const MonitorInterval = 30 * time.Second
|
||||||
|
|
||||||
|
type MemoryMonitor struct {
|
||||||
|
peakAlloc atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewMemoryMonitor() *MemoryMonitor {
|
||||||
|
return &MemoryMonitor{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot reads current heap allocation, updates the peak, and returns both.
|
||||||
|
func (m *MemoryMonitor) Snapshot() (currentAlloc, peakAlloc uint64) {
|
||||||
|
var ms runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&ms)
|
||||||
|
currentAlloc = ms.HeapAlloc
|
||||||
|
for {
|
||||||
|
peak := m.peakAlloc.Load()
|
||||||
|
if currentAlloc <= peak {
|
||||||
|
return currentAlloc, peak
|
||||||
|
}
|
||||||
|
if m.peakAlloc.CompareAndSwap(peak, currentAlloc) {
|
||||||
|
return currentAlloc, currentAlloc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts a periodic monitor loop that calls logFn with each snapshot.
|
||||||
|
// It blocks until ctx is cancelled.
|
||||||
|
func (m *MemoryMonitor) Run(ctx context.Context, logFn func(currentAlloc, peakAlloc uint64)) {
|
||||||
|
ticker := time.NewTicker(MonitorInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
current, peak := m.Snapshot()
|
||||||
|
logFn(current, peak)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultLogFn returns a log function with the given prefix.
|
||||||
|
func DefaultLogFn(prefix string) func(currentAlloc, peakAlloc uint64) {
|
||||||
|
return func(currentAlloc, peakAlloc uint64) {
|
||||||
|
log.Printf("[%s] heap alloc: %s | peak heap alloc: %s",
|
||||||
|
prefix, FormatBytes(currentAlloc), FormatBytes(peakAlloc))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,12 +4,14 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
custom_rwmutex "git.neurocipta.com/rogerferdinan/custom-rwmutex"
|
customrwmutex "git.neurocipta.com/rogerferdinan/custom-rwmutex"
|
||||||
|
safemap "git.neurocipta.com/rogerferdinan/safe-map"
|
||||||
"git.neurocipta.com/rogerferdinan/safe-web-socket/internal"
|
"git.neurocipta.com/rogerferdinan/safe-web-socket/internal"
|
||||||
"github.com/gorilla/websocket"
|
"github.com/gorilla/websocket"
|
||||||
)
|
)
|
||||||
@@ -17,61 +19,16 @@ import (
|
|||||||
const (
|
const (
|
||||||
pingPeriod = 10 * time.Second
|
pingPeriod = 10 * time.Second
|
||||||
readDeadline = 30 * time.Second
|
readDeadline = 30 * time.Second
|
||||||
|
writeDeadline = 10 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
type SafeMap[K comparable, V any] struct {
|
|
||||||
m sync.Map
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewSafeMap[K comparable, V any]() *SafeMap[K, V] {
|
|
||||||
return &SafeMap[K, V]{
|
|
||||||
m: sync.Map{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SafeMap[K, V]) Store(key K, value V) {
|
|
||||||
sm.m.Store(key, value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SafeMap[K, V]) Load(key K) (value V, ok bool) {
|
|
||||||
val, loaded := sm.m.Load(key)
|
|
||||||
if !loaded {
|
|
||||||
return *new(V), false
|
|
||||||
}
|
|
||||||
return val.(V), true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SafeMap[K, V]) Delete(key K) {
|
|
||||||
sm.m.Delete(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SafeMap[K, V]) Range(f func(K, V) bool) {
|
|
||||||
sm.m.Range(func(key, value any) bool {
|
|
||||||
k, ok1 := key.(K)
|
|
||||||
v, ok2 := value.(V)
|
|
||||||
if !ok1 || !ok2 {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return f(k, v)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SafeMap[K, V]) Len() int {
|
|
||||||
count := 0
|
|
||||||
sm.Range(func(_ K, _ V) bool {
|
|
||||||
count++
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
return count
|
|
||||||
}
|
|
||||||
|
|
||||||
type MessageType uint
|
type MessageType uint
|
||||||
|
|
||||||
const (
|
const (
|
||||||
MessageTypeText MessageType = iota + 1
|
MessageTypeText MessageType = websocket.TextMessage
|
||||||
MessageTypePing MessageType = iota
|
MessageTypePing MessageType = websocket.PingMessage
|
||||||
MessageTypePong MessageType = iota
|
MessageTypePong MessageType = websocket.PongMessage
|
||||||
MessageTypeClose MessageType = iota
|
MessageTypeClose MessageType = websocket.CloseMessage
|
||||||
)
|
)
|
||||||
|
|
||||||
type Message struct {
|
type Message struct {
|
||||||
@@ -82,11 +39,13 @@ type Message struct {
|
|||||||
type SafeWebsocketClientBuilder struct {
|
type SafeWebsocketClientBuilder struct {
|
||||||
baseHost *string `nil_checker:"required"`
|
baseHost *string `nil_checker:"required"`
|
||||||
basePort *uint16 `nil_checker:"required"`
|
basePort *uint16 `nil_checker:"required"`
|
||||||
|
headers *map[string]string
|
||||||
path *string
|
path *string
|
||||||
rawQuery *string
|
rawQuery *string
|
||||||
|
isDrop *bool
|
||||||
useTLS *bool
|
useTLS *bool
|
||||||
channelSize *int64
|
channelSize *int64
|
||||||
authenticateFn func(*SafeWebsocketClient) error
|
writeChannelSize *int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewSafeWebsocketClientBuilder() *SafeWebsocketClientBuilder {
|
func NewSafeWebsocketClientBuilder() *SafeWebsocketClientBuilder {
|
||||||
@@ -103,13 +62,13 @@ func (b *SafeWebsocketClientBuilder) BasePort(port uint16) *SafeWebsocketClientB
|
|||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *SafeWebsocketClientBuilder) UseTLS(useTLS bool) *SafeWebsocketClientBuilder {
|
func (b *SafeWebsocketClientBuilder) Headers(headers map[string]string) *SafeWebsocketClientBuilder {
|
||||||
b.useTLS = &useTLS
|
b.headers = &headers
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *SafeWebsocketClientBuilder) AuthenticateFn(authenticateFn func(*SafeWebsocketClient) error) *SafeWebsocketClientBuilder {
|
func (b *SafeWebsocketClientBuilder) UseTLS(useTLS bool) *SafeWebsocketClientBuilder {
|
||||||
b.authenticateFn = authenticateFn
|
b.useTLS = &useTLS
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,19 +82,35 @@ func (b *SafeWebsocketClientBuilder) RawQuery(rawQuery string) *SafeWebsocketCli
|
|||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *SafeWebsocketClientBuilder) IsDrop(isDrop bool) *SafeWebsocketClientBuilder {
|
||||||
|
b.isDrop = &isDrop
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
func (b *SafeWebsocketClientBuilder) ChannelSize(channelSize int64) *SafeWebsocketClientBuilder {
|
func (b *SafeWebsocketClientBuilder) ChannelSize(channelSize int64) *SafeWebsocketClientBuilder {
|
||||||
b.channelSize = &channelSize
|
b.channelSize = &channelSize
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *SafeWebsocketClientBuilder) WriteChannelSize(writeChannelSize int64) *SafeWebsocketClientBuilder {
|
||||||
|
b.writeChannelSize = &writeChannelSize
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
func (b *SafeWebsocketClientBuilder) Build(ctx context.Context) (*SafeWebsocketClient, error) {
|
func (b *SafeWebsocketClientBuilder) Build(ctx context.Context) (*SafeWebsocketClient, error) {
|
||||||
if err := internal.NilChecker(b); err != nil {
|
if err := internal.NilChecker(b); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var useTLS bool
|
// var useTLS bool
|
||||||
if b.useTLS != nil {
|
if b.useTLS == nil {
|
||||||
useTLS = *b.useTLS
|
useTLS := true
|
||||||
|
b.useTLS = &useTLS
|
||||||
|
}
|
||||||
|
|
||||||
|
if b.isDrop == nil {
|
||||||
|
isDrop := true
|
||||||
|
b.isDrop = &isDrop
|
||||||
}
|
}
|
||||||
|
|
||||||
if b.channelSize == nil {
|
if b.channelSize == nil {
|
||||||
@@ -143,26 +118,31 @@ func (b *SafeWebsocketClientBuilder) Build(ctx context.Context) (*SafeWebsocketC
|
|||||||
b.channelSize = &channelSize
|
b.channelSize = &channelSize
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if b.writeChannelSize == nil {
|
||||||
|
writeChannelSize := int64(1)
|
||||||
|
b.writeChannelSize = &writeChannelSize
|
||||||
|
}
|
||||||
|
|
||||||
wsClient := SafeWebsocketClient{
|
wsClient := SafeWebsocketClient{
|
||||||
baseHost: *b.baseHost,
|
baseHost: *b.baseHost,
|
||||||
basePort: *b.basePort,
|
basePort: *b.basePort,
|
||||||
useTLS: useTLS,
|
headers: b.headers,
|
||||||
|
useTLS: *b.useTLS,
|
||||||
|
isDrop: *b.isDrop,
|
||||||
path: b.path,
|
path: b.path,
|
||||||
rawQuery: b.rawQuery,
|
rawQuery: b.rawQuery,
|
||||||
dataChannel: make(chan []byte, *b.channelSize),
|
dataChannel: make(chan []byte, *b.channelSize),
|
||||||
mu: custom_rwmutex.NewCustomRwMutex(),
|
mu: customrwmutex.NewCustomRwMutex(),
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
reconnectCh: make(chan struct{}, 1),
|
reconnectCh: make(chan struct{}, 1),
|
||||||
isConnected: false,
|
isConnected: false,
|
||||||
doneMap: NewSafeMap[string, chan struct{}](),
|
doneMap: safemap.NewSafeMap[string, chan struct{}](),
|
||||||
writeChan: make(chan Message),
|
writeChan: make(chan Message, *b.writeChannelSize),
|
||||||
}
|
monitor: internal.NewMemoryMonitor(),
|
||||||
|
|
||||||
if b.authenticateFn != nil {
|
|
||||||
wsClient.authenticateFn = b.authenticateFn
|
|
||||||
}
|
}
|
||||||
|
|
||||||
go wsClient.reconnectHandler()
|
go wsClient.reconnectHandler()
|
||||||
|
go wsClient.monitor.Run(ctx, internal.DefaultLogFn("client-monitor"))
|
||||||
|
|
||||||
if err := wsClient.connect(); err != nil {
|
if err := wsClient.connect(); err != nil {
|
||||||
return nil, fmt.Errorf("failed to establish initial connection: %v", err)
|
return nil, fmt.Errorf("failed to establish initial connection: %v", err)
|
||||||
@@ -174,27 +154,30 @@ func (b *SafeWebsocketClientBuilder) Build(ctx context.Context) (*SafeWebsocketC
|
|||||||
type SafeWebsocketClient struct {
|
type SafeWebsocketClient struct {
|
||||||
baseHost string
|
baseHost string
|
||||||
basePort uint16
|
basePort uint16
|
||||||
|
headers *map[string]string
|
||||||
|
|
||||||
|
isDrop bool
|
||||||
useTLS bool
|
useTLS bool
|
||||||
path *string
|
path *string
|
||||||
rawQuery *string
|
rawQuery *string
|
||||||
|
|
||||||
dataChannel chan []byte
|
mu *customrwmutex.CustomRwMutex
|
||||||
mu *custom_rwmutex.CustomRwMutex
|
|
||||||
conn *websocket.Conn
|
conn *websocket.Conn
|
||||||
cancelFuncs []context.CancelFunc
|
|
||||||
|
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
Cancel context.CancelFunc
|
cancelFuncs []context.CancelFunc
|
||||||
|
dataChannel chan []byte
|
||||||
|
dataChannelOnce sync.Once
|
||||||
|
|
||||||
reconnectCh chan struct{}
|
reconnectCh chan struct{}
|
||||||
reconnectChans []chan struct{}
|
reconnectChans []chan struct{}
|
||||||
isConnected bool
|
isConnected bool
|
||||||
doneMap *SafeMap[string, chan struct{}]
|
doneMap *safemap.SafeMap[string, chan struct{}]
|
||||||
authenticateFn func(*SafeWebsocketClient) error
|
|
||||||
|
|
||||||
writeChan chan Message
|
writeChan chan Message
|
||||||
|
monitor *internal.MemoryMonitor
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) connect() error {
|
func (wsClient *SafeWebsocketClient) connect() error {
|
||||||
var scheme string
|
var scheme string
|
||||||
if wsClient.useTLS {
|
if wsClient.useTLS {
|
||||||
@@ -206,172 +189,92 @@ func (wsClient *SafeWebsocketClient) connect() error {
|
|||||||
Scheme: scheme,
|
Scheme: scheme,
|
||||||
Host: fmt.Sprintf("%s:%d", wsClient.baseHost, wsClient.basePort),
|
Host: fmt.Sprintf("%s:%d", wsClient.baseHost, wsClient.basePort),
|
||||||
}
|
}
|
||||||
|
|
||||||
if wsClient.path != nil && strings.TrimSpace(*wsClient.path) != "" {
|
if wsClient.path != nil && strings.TrimSpace(*wsClient.path) != "" {
|
||||||
newURL.Path = *wsClient.path
|
newURL.Path = *wsClient.path
|
||||||
}
|
}
|
||||||
|
|
||||||
if wsClient.rawQuery != nil && strings.TrimSpace(*wsClient.rawQuery) != "" {
|
if wsClient.rawQuery != nil && strings.TrimSpace(*wsClient.rawQuery) != "" {
|
||||||
newURL.RawQuery = *wsClient.rawQuery
|
newURL.RawQuery = *wsClient.rawQuery
|
||||||
}
|
}
|
||||||
|
|
||||||
conn, _, err := websocket.DefaultDialer.Dial(newURL.String(), nil)
|
header := make(http.Header, 0)
|
||||||
|
if wsClient.headers != nil {
|
||||||
|
for k, v := range *wsClient.headers {
|
||||||
|
header.Set(k, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, _, err := websocket.DefaultDialer.Dial(newURL.String(), header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to connect to %s: %w", wsClient.baseHost, err)
|
return fmt.Errorf("failed to connect to %s: %w", wsClient.baseHost, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
conn.SetPingHandler(func(pingData string) error {
|
|
||||||
if err := conn.WriteMessage(websocket.PongMessage, []byte(pingData)); err != nil {
|
|
||||||
if err == websocket.ErrCloseSent {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if netErr, ok := err.(interface{ Timeout() bool }); ok && netErr.Timeout() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
pingCtx, pingCancel := context.WithCancel(context.Background())
|
pingCtx, pingCancel := context.WithCancel(context.Background())
|
||||||
|
pumpCtx, pumpCancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
wsClient.mu.WriteHandler(func() error {
|
wsClient.mu.WriteHandler(func() error {
|
||||||
wsClient.cancelFuncs = append(wsClient.cancelFuncs, pingCancel)
|
if wsClient.conn != nil {
|
||||||
|
wsClient.conn.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
wsClient.conn = conn
|
||||||
|
|
||||||
|
wsClient.cancelFuncs = append(wsClient.cancelFuncs, pingCancel, pumpCancel)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
go wsClient.startPingTicker(pingCtx)
|
go wsClient.startPingTicker(pingCtx)
|
||||||
|
go wsClient.writePump(pumpCtx, conn)
|
||||||
|
go wsClient.readPump(pumpCtx, conn)
|
||||||
|
|
||||||
if wsClient.conn != nil {
|
conn.SetPingHandler(func(pingData string) error {
|
||||||
wsClient.conn.Close()
|
|
||||||
}
|
|
||||||
wsClient.conn = conn
|
|
||||||
wsClient.isConnected = true
|
|
||||||
|
|
||||||
if wsClient.authenticateFn != nil {
|
|
||||||
if err := wsClient.authenticateFn(wsClient); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
wsClient.mu.WriteHandler(func() error {
|
|
||||||
wsClient.cancelFuncs = append(wsClient.cancelFuncs, cancel)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
log.Println("Writer stopped due to client shutdown")
|
|
||||||
return
|
|
||||||
case data := <-wsClient.writeChan:
|
|
||||||
if conn == nil {
|
|
||||||
wsClient.triggerReconnect()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := conn.WriteMessage(int(data.MessageType), data.Data); err != nil {
|
|
||||||
log.Printf("error on write message: %v\n", err)
|
|
||||||
wsClient.triggerReconnect()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
wsClient.mu.WriteHandler(func() error {
|
|
||||||
wsClient.cancelFuncs = append(wsClient.cancelFuncs, cancel)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
log.Println("Reader stopped due to client shutdown")
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
if conn == nil {
|
|
||||||
wsClient.triggerReconnect()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil {
|
if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil {
|
||||||
log.Printf("error on read deadline: %v\n", err)
|
log.Printf("error on read deadline: %v\n", err)
|
||||||
return
|
return err
|
||||||
}
|
|
||||||
|
|
||||||
messageType, data, err := conn.ReadMessage()
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error on read message: %v\n", err)
|
|
||||||
wsClient.triggerReconnect()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if messageType != websocket.TextMessage {
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case wsClient.dataChannel <- data:
|
case wsClient.writeChan <- Message{
|
||||||
case <-ctx.Done():
|
MessageType: MessageTypePong,
|
||||||
return
|
Data: []byte(pingData),
|
||||||
|
}:
|
||||||
default:
|
default:
|
||||||
log.Println("Data channel full, dropping message")
|
log.Println("writeChan full, dropping pong")
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
})
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) startPingTicker(ctx context.Context) {
|
conn.SetPongHandler(func(pingData string) error {
|
||||||
ticker := time.NewTicker(pingPeriod)
|
if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil {
|
||||||
defer ticker.Stop()
|
log.Printf("error on read deadline: %v\n", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
for {
|
wsClient.isConnected = true
|
||||||
select {
|
return nil
|
||||||
case <-ctx.Done():
|
|
||||||
log.Println("ping ticker stopped")
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
wsClient.writeChan <- Message{
|
|
||||||
MessageType: websocket.PingMessage,
|
|
||||||
Data: nil,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) triggerReconnect() {
|
|
||||||
select {
|
|
||||||
case wsClient.reconnectCh <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) reconnectHandler() {
|
func (wsClient *SafeWebsocketClient) reconnectHandler() {
|
||||||
backoff := 1 * time.Second
|
backoff := 1 * time.Second
|
||||||
maxBackoff := 30 * time.Second
|
maxBackoff := 15 * time.Second
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-wsClient.reconnectCh:
|
case <-wsClient.reconnectCh:
|
||||||
log.Println("Reconnect triggered")
|
log.Println("Reconnect triggered")
|
||||||
|
|
||||||
wsClient.mu.ReadHandler(func() error {
|
wsClient.mu.WriteHandler(func() error {
|
||||||
if wsClient.cancelFuncs != nil {
|
if wsClient.cancelFuncs != nil {
|
||||||
for _, cancel := range wsClient.cancelFuncs {
|
for _, cancel := range wsClient.cancelFuncs {
|
||||||
cancel()
|
cancel()
|
||||||
}
|
}
|
||||||
|
wsClient.cancelFuncs = nil
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
wsClient.isConnected = false
|
wsClient.isConnected = false
|
||||||
time.Sleep(100 * time.Millisecond)
|
|
||||||
|
|
||||||
isInnerLoop := true
|
isInnerLoop := true
|
||||||
for isInnerLoop {
|
for isInnerLoop {
|
||||||
log.Printf("Attempting reconnect in %v...", backoff)
|
log.Printf("Attempting reconnect in %v...", backoff)
|
||||||
@@ -396,9 +299,13 @@ func (wsClient *SafeWebsocketClient) reconnectHandler() {
|
|||||||
}
|
}
|
||||||
if wsClient.reconnectChans != nil {
|
if wsClient.reconnectChans != nil {
|
||||||
for _, reconnectCh := range wsClient.reconnectChans {
|
for _, reconnectCh := range wsClient.reconnectChans {
|
||||||
reconnectCh <- struct{}{}
|
select {
|
||||||
|
case reconnectCh <- struct{}{}:
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
wsClient.reconnectChans = nil
|
||||||
|
}
|
||||||
case <-wsClient.ctx.Done():
|
case <-wsClient.ctx.Done():
|
||||||
log.Println("reconnect handler stopped due to client shutdown")
|
log.Println("reconnect handler stopped due to client shutdown")
|
||||||
wsClient.Close()
|
wsClient.Close()
|
||||||
@@ -407,8 +314,108 @@ func (wsClient *SafeWebsocketClient) reconnectHandler() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (wsClient *SafeWebsocketClient) writePump(ctx context.Context, c *websocket.Conn) {
|
||||||
|
defer func() {
|
||||||
|
c.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.Println("Writer canceled by context")
|
||||||
|
return
|
||||||
|
case data := <-wsClient.writeChan:
|
||||||
|
if err := c.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil {
|
||||||
|
log.Printf("error setting write deadline: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.WriteMessage(int(data.MessageType), data.Data); err != nil {
|
||||||
|
log.Printf("error on write message: %v\n", err)
|
||||||
|
wsClient.triggerReconnect() // Trigger reconnect on write failure
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (wsClient *SafeWebsocketClient) readPump(ctx context.Context, c *websocket.Conn) {
|
||||||
|
canceled := false
|
||||||
|
defer func() {
|
||||||
|
if !canceled {
|
||||||
|
wsClient.triggerReconnect()
|
||||||
|
}
|
||||||
|
c.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.Println("Reader canceled by context")
|
||||||
|
canceled = true
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
// Set read deadline
|
||||||
|
if err := c.SetReadDeadline(time.Now().Add(readDeadline)); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
messageType, data, err := c.ReadMessage()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error on read message: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if messageType != websocket.TextMessage {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case wsClient.dataChannel <- data:
|
||||||
|
case <-ctx.Done():
|
||||||
|
canceled = true
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
if wsClient.isDrop {
|
||||||
|
log.Println("Data channel full, dropping message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (wsClient *SafeWebsocketClient) startPingTicker(ctx context.Context) {
|
||||||
|
ticker := time.NewTicker(pingPeriod)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.Println("ping ticker canceled by context")
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
select {
|
||||||
|
case wsClient.writeChan <- Message{
|
||||||
|
MessageType: websocket.PingMessage,
|
||||||
|
Data: []byte{},
|
||||||
|
}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.Println("ping ticker canceled by context")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (wsClient *SafeWebsocketClient) triggerReconnect() {
|
||||||
|
select {
|
||||||
|
case wsClient.reconnectCh <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) ReconnectChannel() <-chan struct{} {
|
func (wsClient *SafeWebsocketClient) ReconnectChannel() <-chan struct{} {
|
||||||
reconnectCh := make(chan struct{})
|
reconnectCh := make(chan struct{}, 1)
|
||||||
wsClient.mu.WriteHandler(func() error {
|
wsClient.mu.WriteHandler(func() error {
|
||||||
wsClient.reconnectChans = append(wsClient.reconnectChans, reconnectCh)
|
wsClient.reconnectChans = append(wsClient.reconnectChans, reconnectCh)
|
||||||
return nil
|
return nil
|
||||||
@@ -421,6 +428,12 @@ func (wsClient *SafeWebsocketClient) DataChannel() <-chan []byte {
|
|||||||
return wsClient.dataChannel
|
return wsClient.dataChannel
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (wsClient *SafeWebsocketClient) CloseDataChannel() {
|
||||||
|
wsClient.dataChannelOnce.Do(func() {
|
||||||
|
close(wsClient.dataChannel)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) Write(data []byte) error {
|
func (wsClient *SafeWebsocketClient) Write(data []byte) error {
|
||||||
wsClient.writeChan <- Message{
|
wsClient.writeChan <- Message{
|
||||||
MessageType: MessageTypeText,
|
MessageType: MessageTypeText,
|
||||||
@@ -430,7 +443,7 @@ func (wsClient *SafeWebsocketClient) Write(data []byte) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (wsClient *SafeWebsocketClient) Close() error {
|
func (wsClient *SafeWebsocketClient) Close() error {
|
||||||
wsClient.mu.ReadHandler(func() error {
|
wsClient.mu.WriteHandler(func() error {
|
||||||
if wsClient.cancelFuncs != nil {
|
if wsClient.cancelFuncs != nil {
|
||||||
for _, cancel := range wsClient.cancelFuncs {
|
for _, cancel := range wsClient.cancelFuncs {
|
||||||
cancel()
|
cancel()
|
||||||
@@ -448,7 +461,10 @@ func (wsClient *SafeWebsocketClient) Close() error {
|
|||||||
wsClient.conn.Close()
|
wsClient.conn.Close()
|
||||||
}
|
}
|
||||||
wsClient.isConnected = false
|
wsClient.isConnected = false
|
||||||
// close(wsClient.dataChannel)
|
|
||||||
|
wsClient.dataChannelOnce.Do(func() {
|
||||||
|
close(wsClient.dataChannel)
|
||||||
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,10 +8,18 @@ import (
|
|||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
|
"net/http"
|
||||||
|
_ "net/http/pprof"
|
||||||
|
|
||||||
"git.neurocipta.com/rogerferdinan/safe-web-socket/v1/client"
|
"git.neurocipta.com/rogerferdinan/safe-web-socket/v1/client"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
go func() {
|
||||||
|
log.Println("Starting pprof server on :6060")
|
||||||
|
log.Println(http.ListenAndServe(":6060", nil))
|
||||||
|
}()
|
||||||
|
|
||||||
sigChan := make(chan os.Signal, 1)
|
sigChan := make(chan os.Signal, 1)
|
||||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
|
||||||
@@ -27,9 +35,11 @@ func main() {
|
|||||||
wsClient, err := client.NewSafeWebsocketClientBuilder().
|
wsClient, err := client.NewSafeWebsocketClientBuilder().
|
||||||
BaseHost("localhost").
|
BaseHost("localhost").
|
||||||
BasePort(8080).
|
BasePort(8080).
|
||||||
Path("/ws/test/data_1").
|
Headers(map[string]string{
|
||||||
|
"X-MBX-APIKEY": "abcd",
|
||||||
|
}).Path("/ws/test/data_1").
|
||||||
UseTLS(false).
|
UseTLS(false).
|
||||||
ChannelSize(30).
|
ChannelSize(1).
|
||||||
Build(ctx)
|
Build(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
@@ -44,7 +54,7 @@ func main() {
|
|||||||
dataChannel := wsClient.DataChannel()
|
dataChannel := wsClient.DataChannel()
|
||||||
|
|
||||||
for data := range dataChannel {
|
for data := range dataChannel {
|
||||||
// _ = data
|
_ = data
|
||||||
fmt.Println(string(data))
|
// fmt.Println(string(data))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
28
v1/examples/gobwas-server/main.go
Normal file
28
v1/examples/gobwas-server/main.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func setMaxRLimit() {
|
||||||
|
var rLimit syscall.Rlimit
|
||||||
|
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
rLimit.Cur = rLimit.Max
|
||||||
|
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
setMaxRLimit()
|
||||||
|
// pooler, err := netpool.New()
|
||||||
|
|
||||||
|
// http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// if _, _, _, err := ws.UpgradeHTTP(r, w); err != nil {
|
||||||
|
// log.Fatal(err)
|
||||||
|
// }
|
||||||
|
// })
|
||||||
|
// epoller, err := MkEp
|
||||||
|
}
|
||||||
@@ -1,8 +1,12 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log"
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.neurocipta.com/rogerferdinan/safe-web-socket/v1/server"
|
"git.neurocipta.com/rogerferdinan/safe-web-socket/v1/server"
|
||||||
@@ -14,12 +18,30 @@ type ExampleData struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
sigChan := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
go func() {
|
||||||
|
<-sigChan
|
||||||
|
log.Println("Received shutdown signal")
|
||||||
|
cancel()
|
||||||
|
}()
|
||||||
|
|
||||||
s, err := server.NewSafeWebsocketServerBuilder().
|
s, err := server.NewSafeWebsocketServerBuilder().
|
||||||
BaseHost("localhost").
|
BaseHost("localhost").
|
||||||
BasePort(8080).
|
BasePort(8080).
|
||||||
HandleFuncWebsocket("/ws/test/", "data_1", func(c chan []byte) {
|
ApiKey("abcd").
|
||||||
ticker := time.NewTicker(10 * time.Millisecond)
|
Context(ctx).
|
||||||
for range ticker.C {
|
HandleFuncWebsocket("/ws/test/", "data_1", 5_000, func(ctx context.Context, c chan []byte) {
|
||||||
|
ticker := time.NewTicker(100 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
jsonBytes, err := json.Marshal(ExampleData{
|
jsonBytes, err := json.Marshal(ExampleData{
|
||||||
Time: time.Now(),
|
Time: time.Now(),
|
||||||
Data: "data_1",
|
Data: "data_1",
|
||||||
@@ -27,12 +49,22 @@ func main() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
c <- jsonBytes
|
select {
|
||||||
|
case c <- jsonBytes:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}).
|
}).
|
||||||
HandleFuncWebsocket("/ws/test/", "data_2", func(c chan []byte) {
|
HandleFuncWebsocket("/ws/test/", "data_2", 5_000, func(ctx context.Context, c chan []byte) {
|
||||||
ticker := time.NewTicker(10 * time.Millisecond)
|
ticker := time.NewTicker(100 * time.Millisecond)
|
||||||
for range ticker.C {
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
jsonBytes, err := json.Marshal(ExampleData{
|
jsonBytes, err := json.Marshal(ExampleData{
|
||||||
Time: time.Now(),
|
Time: time.Now(),
|
||||||
Data: "data_2",
|
Data: "data_2",
|
||||||
@@ -40,7 +72,12 @@ func main() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
c <- jsonBytes
|
select {
|
||||||
|
case c <- jsonBytes:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}).
|
}).
|
||||||
Build()
|
Build()
|
||||||
|
|||||||
@@ -1,20 +1,37 @@
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/subtle"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.neurocipta.com/rogerferdinan/safe-web-socket/internal"
|
"git.neurocipta.com/rogerferdinan/safe-web-socket/internal"
|
||||||
"github.com/gorilla/websocket"
|
"github.com/gorilla/websocket"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func setMaxRLimit() {
|
||||||
|
var rLimit syscall.Rlimit
|
||||||
|
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
rLimit.Cur = rLimit.Max
|
||||||
|
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type SafeWebsocketServerBuilder struct {
|
type SafeWebsocketServerBuilder struct {
|
||||||
baseHost *string `nil_checker:"required"`
|
baseHost *string `nil_checker:"required"`
|
||||||
basePort *uint16 `nil_checker:"required"`
|
basePort *uint16 `nil_checker:"required"`
|
||||||
|
apiKey *string `nil_checker:"required"`
|
||||||
upgrader *websocket.Upgrader `nil_checker:"required"`
|
upgrader *websocket.Upgrader `nil_checker:"required"`
|
||||||
mux *http.ServeMux `nil_checker:"required"`
|
mux *http.ServeMux `nil_checker:"required"`
|
||||||
|
ctx context.Context
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewSafeWebsocketServerBuilder() *SafeWebsocketServerBuilder {
|
func NewSafeWebsocketServerBuilder() *SafeWebsocketServerBuilder {
|
||||||
@@ -27,6 +44,7 @@ func NewSafeWebsocketServerBuilder() *SafeWebsocketServerBuilder {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
mux: http.NewServeMux(),
|
mux: http.NewServeMux(),
|
||||||
|
ctx: context.Background(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,18 +58,31 @@ func (b *SafeWebsocketServerBuilder) BasePort(basePort uint16) *SafeWebsocketSer
|
|||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *SafeWebsocketServerBuilder) HandleFunc(
|
func (b *SafeWebsocketServerBuilder) ApiKey(apiKey string) *SafeWebsocketServerBuilder {
|
||||||
pattern string, fn func(http.ResponseWriter, *http.Request),
|
b.apiKey = &apiKey
|
||||||
) *SafeWebsocketServerBuilder {
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// Context sets the lifecycle context for all hub and writeFunc goroutines.
|
||||||
|
// When ctx is cancelled, hubs stop dispatching and all active connections
|
||||||
|
// are unblocked so their goroutines can exit cleanly.
|
||||||
|
// Call this before HandleFuncWebsocket. Defaults to context.Background().
|
||||||
|
func (b *SafeWebsocketServerBuilder) Context(ctx context.Context) *SafeWebsocketServerBuilder {
|
||||||
|
b.ctx = ctx
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *SafeWebsocketServerBuilder) HandleFunc(pattern string, fn func(http.ResponseWriter, *http.Request)) *SafeWebsocketServerBuilder {
|
||||||
b.mux.HandleFunc(pattern, fn)
|
b.mux.HandleFunc(pattern, fn)
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *SafeWebsocketServerBuilder) HandleFuncWebsocket(
|
// HandleFuncWebsocket registers a WebSocket endpoint.
|
||||||
pattern string, subscribedPath string, writeFunc func(chan []byte),
|
func (b *SafeWebsocketServerBuilder) HandleFuncWebsocket(pattern string, subscribedPath string, maxClients int, writeFunc func(ctx context.Context, writeChannel chan []byte)) *SafeWebsocketServerBuilder {
|
||||||
) *SafeWebsocketServerBuilder {
|
h := internal.NewHub(pattern+subscribedPath, maxClients)
|
||||||
h := internal.NewHub()
|
h.Run(b.ctx)
|
||||||
h.Run()
|
|
||||||
|
go writeFunc(b.ctx, h.Broadcast)
|
||||||
|
|
||||||
b.mux.HandleFunc(pattern+subscribedPath, func(w http.ResponseWriter, r *http.Request) {
|
b.mux.HandleFunc(pattern+subscribedPath, func(w http.ResponseWriter, r *http.Request) {
|
||||||
conn, err := b.upgrader.Upgrade(w, r, nil)
|
conn, err := b.upgrader.Upgrade(w, r, nil)
|
||||||
@@ -67,9 +98,9 @@ func (b *SafeWebsocketServerBuilder) HandleFuncWebsocket(
|
|||||||
}
|
}
|
||||||
c := internal.NewClient(conn, subscribedPath)
|
c := internal.NewClient(conn, subscribedPath)
|
||||||
h.Register <- c
|
h.Register <- c
|
||||||
|
|
||||||
go internal.WritePump(c, h)
|
go internal.WritePump(c, h)
|
||||||
go internal.ReadPump(c, h)
|
go internal.ReadPump(c, h)
|
||||||
go writeFunc(h.Broadcast)
|
|
||||||
})
|
})
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
@@ -79,23 +110,65 @@ func (b *SafeWebsocketServerBuilder) Build() (*SafeWebsocketServer, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setMaxRLimit()
|
||||||
safeServer := SafeWebsocketServer{
|
safeServer := SafeWebsocketServer{
|
||||||
url: fmt.Sprintf("%s:%d", *b.baseHost, *b.basePort),
|
|
||||||
mux: b.mux,
|
mux: b.mux,
|
||||||
|
url: fmt.Sprintf("%s:%d", *b.baseHost, *b.basePort),
|
||||||
|
apiKey: *b.apiKey,
|
||||||
|
ctx: b.ctx,
|
||||||
}
|
}
|
||||||
return &safeServer, nil
|
return &safeServer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type SafeWebsocketServer struct {
|
type SafeWebsocketServer struct {
|
||||||
hub *internal.Hub
|
|
||||||
mux *http.ServeMux
|
mux *http.ServeMux
|
||||||
url string
|
url string
|
||||||
|
apiKey string
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SafeWebsocketServer) AuthMiddleware(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
providedKey := r.Header.Get("X-MBX-APIKEY")
|
||||||
|
expectedKey := s.apiKey
|
||||||
|
|
||||||
|
if subtle.ConstantTimeCompare([]byte(providedKey), []byte(expectedKey)) != 1 {
|
||||||
|
internal.ErrorResponse(w, internal.NewStatusMessage().
|
||||||
|
StatusCode(http.StatusForbidden).
|
||||||
|
Message("X-MBX-APIKEY is missing").
|
||||||
|
Build())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SafeWebsocketServer) ListenAndServe() error {
|
func (s *SafeWebsocketServer) ListenAndServe() error {
|
||||||
log.Printf("HTTP serve on %s\n", s.url)
|
srv := &http.Server{
|
||||||
if err := http.ListenAndServe(s.url, s.mux); err != nil {
|
Addr: s.url,
|
||||||
return fmt.Errorf("failed to serve websocket: %w", err)
|
Handler: s.AuthMiddleware(s.mux),
|
||||||
|
}
|
||||||
|
|
||||||
|
errCh := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
log.Printf("HTTP serve on %s\n", s.url)
|
||||||
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
|
errCh <- fmt.Errorf("failed to serve websocket: %w", err)
|
||||||
|
}
|
||||||
|
close(errCh)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-errCh:
|
||||||
|
return err
|
||||||
|
case <-s.ctx.Done():
|
||||||
|
log.Println("Server shutting down...")
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||||
|
return fmt.Errorf("server shutdown: %w", err)
|
||||||
|
}
|
||||||
|
return <-errCh
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user