From 7dbfa2755c7f0dcbea47b4f5bd3ff119a9ed48a0 Mon Sep 17 00:00:00 2001 From: Tim Connors Date: Sat, 13 Sep 2025 16:05:32 +1000 Subject: [PATCH] Update documentation, comments, logging, and centralise some management Management of ambient/exhaust temperature management delegated to daemon number 0. Hopefully all aspects of the code should be documented now. Logging is a lot nicer --- poweredge-fand.conf | 112 +++++++++++++++++++++++++------------ poweredge-fand.pl | 132 ++++++++++++++++++++++++++++++-------------- 2 files changed, 167 insertions(+), 77 deletions(-) diff --git a/poweredge-fand.conf b/poweredge-fand.conf index 56ac41a..3ed44fd 100755 --- a/poweredge-fand.conf +++ b/poweredge-fand.conf @@ -1,52 +1,89 @@ #!/usr/bin/perl $static_speed_low=0x02; -$static_speed_high=0x12; # this is the speed value at 100% demand +$static_speed_high=0x12; # This is the speed value at 100% demand # ie what we consider the point we don't # really want to get hotter but still - # tolerate + # tolerate. It's quite a bit of a misnomer + # - we keep increasing the speed rapidly + # beyond that up to 0xFF if necessary, but + # it's perhaps a scaling factor. $ipmi_inlet_sensorname ="Inlet Temp"; $ipmi_exhaust_sensorname="Exhaust Temp"; -$default_exhaust_threshold=50; # the ambient temperature we use above - # which we fail back to letting the drac - # control the fans -$base_temp = 24; # no fans when below this temp -$desired_temp1 = 34; # aim to keep the temperature below this -$desired_temp2 = 42; # really ramp up fans above this -$desired_temp3 = 50; # really ramp up fans above this -$demand1 = 5; # prescaled (not taking into effect static_speed_low/high) demand at temp1 -$demand2 = 40; # prescaled (not taking into effect static_speed_low/high) demand at temp2 -$demand3 = 200; # prescaled (not taking into effect static_speed_low/high) demand at temp3 +$default_exhaust_threshold=50; # The exhaust temperature we use, set + # from above, which we fail back to + # letting the drac control the fans +$base_temp = 26; # No fans when below this temp +$desired_temp1 = 36; # Aim to keep the temperature below this +$desired_temp2 = 44; # Really ramp up fans above this +$desired_temp3 = 50; # Really ramp up fans above this +$demand1 = 5; # Prescaled (not taking into effect static_speed_low/high) demand at temp1 +$demand2 = 40; # Prescaled (not taking into effect static_speed_low/high) demand at temp2 +$demand3 = 200; # Prescaled (not taking into effect static_speed_low/high) demand at temp3 -$hysteresis = 2; # don't ramp up velocity unless demand +$hysteresis = 2; # Don't ramp up velocity unless demand # difference is greater than this. Ramp # down ASAP however, to bias quietness, and # thus end up removing noise changes for # just small changes in computing -# we can optionally just tell all fans to stick to the one speed: +# We can optionally just tell all fans to stick to the one speed: # @daemons=(0xff); @daemons=qw(0 1 2 3 4 5); sub custom_temperature_calculation { # FIXME: it would be good to share memory between the 6 daemons and # not call ipmitool 6 times in parallel just to discard the value of - # 4 of them - my $gpu = $sensors_ref->{"amdgpu-pci-0400"}->{"edge"}->{"temp1_input"}; # 0.2*fan3 + 0.5*fan4 + 0.3*fan5 - my $raid_card = raid_controller_temp(); # fan4+fan5; - my $left_cpu = $sensors_ref->{"coretemp-isa-0000"}->{"Package id 0"}->{"temp1_input"}; # fan2; - my $right_cpu = $sensors_ref->{"coretemp-isa-0001"}->{"Package id 1"}->{"temp1_input"}; # fan4; + # 4 of them. + # + # These devices must be present, or the daemon will fail when + # starting. Remove them from the calculation if you don't have one + # (eg, GPU). I manually worked out roughly which fans correspond to + # dropping the temperature of each device, by ramping that fan up to + # a high velocity and observing how quickly the temperature drops. + # + # You should manually run `sensors -j` and ensure that every device + # that has a temperature is somewhow included in the calculation of + # at least one fan speed. Note that drivetemp devices may already + # be included via $*_drives averages below. + my $gpu = $sensors_ref->{"amdgpu-pci-0400"}{"edge"}{"temp1_input"}; # 0.2*fan3 + 0.5*fan4 + 0.3*fan5 + my $raid_card = raid_controller_temp(); # fan4+fan5; + my $left_cpu = $sensors_ref->{"coretemp-isa-0000"}{"Package id 0"}{"temp1_input"}; # fan2; + my $right_cpu = $sensors_ref->{"coretemp-isa-0001"}{"Package id 1"}{"temp1_input"}; # fan4; - # you might want to factor in NVME drives too, that might appear - # through sensors structure as eg: - # $sensors_ref->{"nvme-pci-0100"}->{"temp composite Samsung-EVO-970-1TB"}->{"temp1_input"} - # check with manual invocation of `sensors -j`. You might find - # "temp sensor 2 Samsung-EVO-970-1TB" is uncomfortably higher than - # "temp composite Samsung-EVO-970-1TB" but the composite (heatsink) - # is the bit you're cooling. So monitor that hotter sensor instead - # to add more fan demand + # You might want to factor in NVME drives too, that might appear + # through the `sensors -j` structure as eg: + # $sensors_ref->{"nvme-pci-0100"}{"temp composite Samsung-EVO-970-1TB"}{"temp1_input"} + # + # You might find that while "temp composite Samsung-EVO-970-1TB" + # (heatsink) is the bit you're cooling, "temp sensor 2 + # Samsung-EVO-970-1TB" (maybe the silicon itself) is uncomfortably + # higher than the composite reading. So monitor that hotter sensor + # instead to add more fan demand. + # For the drives, there's no need to map each /dev/sd? onto their + # dynamic positioning inside the chassis. Just code each path as it + # maps from /dev/disk/by-path to how it appears on your backplane + # physically. I used: `dd if=/dev/disk/by-path/.... of=/dev/null` + # to work out the pattern that's appropriate to my LFF R730XD (4x3 + # 3.5" drives at front, 2 2.5" drives at rear; hence why I don't + # know which of 12 and 13 are left or right rear - I can't pull my + # machine out far enough to watch the blinkenlights). The averages + # disregard any drives that are not present or spundown and + # inaccessible to SMART temperature sensing (the daemon will + # fallback to idrac control if you define a mapping where one fan + # doesn't cool *any devices*. If you were sure there was nothing + # that could be cooled from that fan (no PCIe cards, no CPU or RAM + # in right hand socket, no drives on right hand side, no drac, no + # RAID controller, etc), you could just set weighted_temp to 1 (not + # 0 - that's still an error). + + # So let's calculate the average temperatures of drives in each + # vertical arrangement (the rear drives draw their airflow from the + # two leftmost fans, so just average them). If you have 2.5" SFF + # drive backplane, or a 1RU chassis, or..., your calculation will be + # different. my $left_outside_drives = average(hddtemp("/dev/disk/by-path/pci-0000:02:00.0-scsi-0:0:0:0"), # front top SSD hddtemp("/dev/disk/by-path/pci-0000:02:00.0-scsi-0:0:1:0"), # front middle SSD @@ -72,6 +109,7 @@ sub custom_temperature_calculation { print "drives = $left_outside_drives, $left_middle_drives, $right_middle_drives, $right_outside_drives ; $rear_drives\n"; } + # My original mapping to spin up all fans in sync was: # Two thirds weighted CPU temps vs hdd temps, but if the HDD temps # creep above this value, use them exclusively (more important to # keep them cool than the CPUs): @@ -83,15 +121,18 @@ sub custom_temperature_calculation { # the difference between weighted_temp and ambient_temp is small # because ambient_temp is large, then less need to run the fans # because there's still low power demands, to a point, but - # eventually we have to keep all components under a threshold + # eventually we have to keep all components under a threshold, so + # maybe it makes sense to just ignore ambient_temp. - # FIXME: hysteresis + # FIXME: better hysteresis function than what's in set_fans_servo() - # It is more important to keep hdds cool than CPUs. We should put - # different offsets or multipliers on them. As it is, we weight - # drives 3 times more important than CPU in the final weighting + # It is more important to keep both SSDs and HDDs cool than CPUs. + # You can replace cooked CPUs, and they'll throttle before that + # point anyway. We could put different offsets or multipliers on + # them. As it is here, we weight drives 3 times more important than + # CPU in the final weighting. - # contribution from drives alone: + # Contribution needed from each to cool the drives alone: my $fan0_drive_weighted_temp = average($left_outside_drives, $rear_drives); my $fan1_drive_weighted_temp = average($left_outside_drives, $left_middle_drives, $rear_drives); my $fan2_drive_weighted_temp = $left_middle_drives; @@ -100,7 +141,8 @@ sub custom_temperature_calculation { my $fan5_drive_weighted_temp = $right_outside_drives; my @fan_weighted_temp; - # contribution from drives, CPUs, raid controller, GPU: + + # Contribution from drives, CPUs, raid controller, GPU: $fan_weighted_temp[0] = $fan0_drive_weighted_temp; $fan_weighted_temp[1] = $fan1_drive_weighted_temp; $fan_weighted_temp[2] = weighted_average(3,$fan2_drive_weighted_temp, 2,$left_cpu); @@ -108,7 +150,7 @@ sub custom_temperature_calculation { $fan_weighted_temp[4] = weighted_average(3,$fan4_drive_weighted_temp, 1,$gpu, 1,$raid_card, 2,$right_cpu); $fan_weighted_temp[5] = weighted_average(3,$fan5_drive_weighted_temp, 1,$gpu, 1,$raid_card); - # which fans are being controlled by this daemon? 0xff = all fans, + # Which fans are being controlled by this daemon? 0xff = all fans, # 0x00 to 0x05 for individual fans from left to right viewing from the # front diff --git a/poweredge-fand.pl b/poweredge-fand.pl index a2743f8..3a3d7c7 100755 --- a/poweredge-fand.pl +++ b/poweredge-fand.pl @@ -20,24 +20,24 @@ use POSIX ":sys_wait_h"; # for nonblocking read use Time::HiRes qw (sleep); my $static_speed_low; -my $static_speed_high; # this is the speed value at 100% demand +my $static_speed_high; # This is the speed value at 100% demand # ie what we consider the point we don't # really want to get hotter but still # tolerate my ($ipmi_inlet_sensorname, $ipmi_exhaust_sensorname); -my $default_exhaust_threshold; # the exhaust temperature we use above - # which we fail back to letting the drac - # control the fans -my $base_temp; # no fans when below this temp -my $desired_temp1; # aim to keep the temperature below this -my $desired_temp2; # really ramp up fans above this -my $desired_temp3; # really ramp up fans above this -my $demand1; # prescaled (not taking into effect static_speed_low/high) demand at temp1 -my $demand2; # prescaled (not taking into effect static_speed_low/high) demand at temp2 -my $demand3; # prescaled (not taking into effect static_speed_low/high) demand at temp3 +my $default_exhaust_threshold; # The exhaust temperature we use, set + # from above, which we fail back to + # letting the drac control the fans +my $base_temp; # No fans when below this temp +my $desired_temp1; # Aim to keep the temperature below this +my $desired_temp2; # Really ramp up fans above this +my $desired_temp3; # Really ramp up fans above this +my $demand1; # Prescaled (not taking into effect static_speed_low/high) demand at temp1 +my $demand2; # Prescaled (not taking into effect static_speed_low/high) demand at temp2 +my $demand3; # Prescaled (not taking into effect static_speed_low/high) demand at temp3 -my $hysteresis; # don't ramp up velocity unless demand +my $hysteresis; # Don't ramp up velocity unless demand # difference is greater than this. Ramp # down ASAP however, to bias quietness, and # thus end up removing noise changes for @@ -48,17 +48,20 @@ my $fans; # which fans are being controlled by this daemon? 0xff sub custom_temperature_calculation; -# every 20 minutes (enough to establish spin-down), invalidate the +# Every 20 minutes (enough to establish spin-down), invalidate the # cache of the slowly changing hdd temperatures to allow them to be # refreshed my $hdd_poll_interval=1200; -# raid controller is less expensive to poll, and should't change +# Raid controller is less expensive to poll, but also should't change # overly rapidly my $raid_controller_poll_interval=30; -# every 60 seconds, invalidate the cache of the slowly changing +# Every 60 seconds, invalidate the cache of the slowly changing # ambient temperatures to allow them to be refreshed my $ambient_poll_interval=60; my $exhaust_poll_interval=60; +# And put fan back under our control once per minute if someone +# slipped it back into idrac mode, if all conditions are correct +my $manual_mode_reset_interval=60; my $cpu_poll_interval=3; my $sensors_ref; @@ -83,9 +86,14 @@ sub print_usage { my %included_conf_file; sub include { - # http://www.perlmonks.org/?node_id=393426 - #package DB; # causes eval to evaluate the string in the caller's - # scope. Sometimes perl can be truly horrendous + # http://www.perlmonks.org/?node_id=393426 but a bit different to + # the version you can see there now in 2025, or maybe this was the + # version I found that works when you're not trying to write a + # module...? Their code wants &include to be defined as + # &module::include, and circa 2025, I was now seeing scope issues + # without that, but removed `package DB; # causes eval to evaluate + # the string in the caller's scope` and the code started to work + # again. So let's go with that... my ($filename) = @_; my $code; @@ -126,11 +134,18 @@ sub sleep_and_check_for_exit { } } +# Only test true in one of the daemons, for the first fan (or the +# daemon that's controlling all fans simultaneously) +sub perform_only_once { + return ($fans == 0 or $fans == 0xff); +} + # Only print out the stats in one of the daemons, for the first # fan (or the daemon that's controlling all fans simultaneously) sub print_stats_once { - return $print_stats and - ($fans == 0 or $fans == 0xff); + my $res=($print_stats and + perform_only_once()); + return $res; } sub is_num { @@ -298,7 +313,11 @@ sub set_fans_default { $lastfan=undef; # this is an abnormal condition, so always warn about it, even in # quiet mode - print "--> enable dynamic (idrac automatic) fan control\n"; + if (perform_only_once) { + print " --> enable dynamic (idrac automatic) fan control\n"; + } + # any one of the daemons is allowed to independently call + # set_fans_default, so we have to run it for each of them: foreach my $attempt (1..10) { # ipmitool routinely fails, so try up to 10 times since we are # already the failure path, so need to be reliable ourselves @@ -322,14 +341,27 @@ sub set_fans_servo { set_fans_default(); return 0; # we always failed, even if set_fans_default succeeded } - # my $ambient_temp = ambient_temp(); - # print "weighted_temp($fans) = $weighted_temp ; ambient_temp $ambient_temp\n" if $print_stats; + my $ambient_temp = ambient_temp(); my $exhaust_temp = exhaust_temp(); - print "weighted_temp($fans) = $weighted_temp ; exhaust_temp $exhaust_temp\n" if $print_stats; + print "ambient_temp $ambient_temp ; exhaust_temp $exhaust_temp" if print_stats_once; - # take us out of idrac dynamic control, setting to manual control - if ((!defined $current_mode) or ($current_mode ne "set")) { - print "--> disable dynamic fan control\n" if !($quiet and (defined $current_mode) and ($current_mode eq "reset")); + # All fan controllers have to agree to take the fan control out of + # idrac control (or, more correctly, any of the controllers can put + # it back into auto-mode), so we only do the manual invocation from + # one of the controllers, in order not to flood the logs and run + # ipmitool more often than necessary (the reset is done once per + # minute anyway if someone did slip it back into idrac control, but + # then didn't have the authority to move it back under manual + # control) + if (perform_only_once) { + if ((!defined $current_mode) or ($current_mode ne "set")) { + if ($print_stats or + !((defined $current_mode) and ($current_mode eq "reset"))) { + print " --> enable our manual fan control\n"; + } + } else { + print "\n" if $print_stats; + } # ipmitool routinely fails; that's OK, if this fails, want to # return telling caller not to think we've made a change if (system("ipmitool raw 0x30 0x30 0x01 0x00") != 0) { @@ -337,7 +369,6 @@ sub set_fans_servo { } $current_mode="set"; } - my $demand = 0; # sort of starts off with a range roughly 0-255, # which we multiply later to be ranged roughly # between 0-100% of @@ -361,22 +392,31 @@ sub set_fans_servo { # the only possibility left is $weighted_temp < $base_temp # which we've already decided is demand=0 } - printf "demand($fans, %0.2f)", $demand if $print_stats; - $demand = int($static_speed_low + $demand/100*($static_speed_high-$static_speed_low)); - if ($demand>255) { - $demand=255; + + my $demand_out = int($static_speed_low + $demand/100*($static_speed_high-$static_speed_low)); + if ($demand_out>255) { + $demand_out=255; } - printf " -> %i\n", $demand if $print_stats; + my $stats_to_print=sprintf "weighted_temp($fans) = %6.2f ; demand($fans)=%6.2f -> %3i", $weighted_temp, $demand, $demand_out; + $demand = $demand_out; + # ramp down the fans quickly upon lack of demand, don't ramp them up # to tiny spikes of 1 fan unit. FIXME: But should implement long # term smoothing of +/- 1 fan unit - if (!defined $lastfan or - $demand < $lastfan or - $demand > $lastfan + $hysteresis) { - $lastfan = $demand; - $demand = sprintf("0x%x", $demand); + my $demand_has_changed = !defined $lastfan or + $demand < $lastfan or + $demand > $lastfan + $hysteresis; + if ($print_stats or + $demand_has_changed) { + if ($demand_has_changed) { + $lastfan = $demand; + } + $demand = sprintf("0x%02x", $demand); # print "demand = $demand\n"; - print "--> ipmitool raw 0x30 0x30 0x02 $fans $demand\n"; + + # allowed to print out of minute-printing cycle if the demand + # actually changes + print "$stats_to_print --> ipmitool raw 0x30 0x30 0x02 $fans $demand\n"; # ipmitool routinely fails; that's OK, if this fails, want to # return telling caller not to think we've made a change if (system("ipmitool raw 0x30 0x30 0x02 $fans $demand") != 0) { @@ -475,11 +515,17 @@ include $conf_file; $started=1; $SIG{TERM} = $SIG{HUP} = $SIG{INT} = \&signal_handler; +my $first_child=1; foreach my $loop_fan (@daemons) { my $pid; if ($pid=fork) { #parent; $children{$pid}=1; + if ($first_child) { + sleep 20; # give time for the first daemon to gather all + # ambient, hdd etc stats + } + $first_child=0; # keep looping } elsif ($pid==0) { #child; @@ -520,8 +566,10 @@ while () { # if ($ambient_temp > $default_threshold) { my $exhaust_temp = exhaust_temp(); if ($exhaust_temp > $default_exhaust_threshold) { - #print "fallback because of high ambient temperature $ambient_temp > $default_threshold\n"; - print "fallback because of high exhaust temperature $exhaust_temp > $default_exhaust_threshold\n"; + if (perform_only_once) { + #print "fallback because of high ambient temperature $ambient_temp > $default_threshold\n"; + print "fallback because of high exhaust temperature $exhaust_temp > $default_exhaust_threshold\n"; + } if (!set_fans_default()) { # return for next loop without resetting timers and delta change # if that fails @@ -536,7 +584,7 @@ while () { } $print_stats = 0; - if (time - $last_print_stats > 60) { + if (time - $last_print_stats > $manual_mode_reset_interval) { $current_mode="reset"; # just in case the RAC has rebooted, it # will go back into default control, so # make sure we set it appropriately once