While I don’t think this is at all finished, this is running on a Dell R920 with 1.5TB memory. I think the cache dirty ratio might be too high, and still tweaking the net tunables. Remember that in general, tcp offload should be turned off as most ethernet cards are too slow to support what the system and linux can do now a days.
This system is 10gb connected and does heavy NFS traffic.
# Controls the use of TCP syncookies
net.ipv4.tcp_syncookies = 1
net.core.rmem_default = 16777216
net.core.rmem_max = 67108864
net.core.wmem_default = 16777216
net.core.wmem_max = 67108864
sunrpc.tcp_slot_table_entries = 1024
net.ipv4.ipfrag_low_thresh = 262144
net.ipv4.ipfrag_high_thresh = 393216
# Net tuning
net.ipv4.tcp_rmem = 393216 1024000 67108864
net.ipv4.tcp_wmem = 393216 1024000 67108864
##
net.ipv4.tcp_mem = 393216 1024000 67108864
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_sack = 0
net.ipv4.tcp_window_scaling = 1
fs.file-max=327679
fs.aio-max-nr=3145728
vm.page-cluster = 4000
vm.min_free_kbytes = 200000
vm.swappiness = 0
vm.dirty_background_ratio = 10
vm.dirty_expire_centisecs = 4000
vm.dirty_ratio = 30
vm.dirty_writeback_centisecs = 1500
vm.vfs_cache_pressure = 10000
vm.lowmem_reserve_ratio = 512 512 64